🚨🚨 Source LinkedIn Ads: update primary key for Analytics Streams (#36927)

artem1205 · web-flow · commit da0a7faca9e4 · 2024-04-15T18:52:32.000+02:00
Signed-off-by: Artem Inzhyyants &lt;artem.inzhyyants@gmail.com&gt;
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml
@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 137ece28-5434-455c-8f34-69dc3782f451
-  dockerImageTag: 0.8.0
+  dockerImageTag: 1.0.0
   dockerRepository: airbyte/source-linkedin-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads
   githubIssueLabel: source-linkedin-ads
@@ -29,6 +29,25 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  releases:
+    breakingChanges:
+      1.0.0:
+        message: This upgrade brings changes in primary key to *-analytics streams.
+        upgradeDeadline: "2024-04-30"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes:
+              - "ad_campaign_analytics"
+              - "ad_creative_analytics"
+              - "ad_impression_device_analytics"
+              - "ad_member_company_size_analytics"
+              - "ad_member_country_analytics"
+              - "ad_member_job_function_analytics"
+              - "ad_member_job_title_analytics"
+              - "ad_member_industry_analytics"
+              - "ad_member_seniority_analytics"
+              - "ad_member_region_analytics"
+              - "ad_member_company_analytics"
   suggestedStreams:
     streams:
       - accounts
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml b/airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.8.0"
+version = "1.0.0"
 name = "source-linkedin-ads"
 description = "Source implementation for Linkedin Ads."
 authors = [ "Airbyte <contact@airbyte.io>",]
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py
@@ -127,13 +127,15 @@ class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC):
 
     endpoint = "adAnalytics"
     # For Analytics streams, the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc.] + `end_date`
-    primary_key = ["pivotValue", "end_date"]
+    primary_key = ["pivotValues", "end_date"]
     cursor_field = "end_date"
     records_limit = 15000
-    FIELDS_CHUNK_SIZE = 19
+    FIELDS_CHUNK_SIZE = 18
 
     def get_json_schema(self) -> Mapping[str, Any]:
-        return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics")
+        schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics")
+        schema["properties"].update({self.search_param_value: {"type": ["null", "string"]}})
+        return schema
 
     def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs):
         self.user_stream_name = name
@@ -286,6 +288,8 @@ def chunk_analytics_fields(
         for chunk in chunks:
             if "dateRange" not in chunk:
                 chunk.append("dateRange")
+            if "pivotValues" not in chunk:
+                chunk.append("pivotValues")
         yield from chunks
 
     def read_records(
@@ -294,15 +298,15 @@ def read_records(
         merged_records = defaultdict(dict)
         for field_slice in stream_slice:
             for rec in super().read_records(stream_slice=field_slice, **kwargs):
-                merged_records[rec[self.cursor_field]].update(rec)
+                merged_records[f"{rec[self.cursor_field]}-{rec['pivotValues']}"].update(rec)
         yield from merged_records.values()
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
         """
         We need to get out the nested complex data structures for further normalization, so the transform_data method is applied.
         """
         for rec in transform_data(response.json().get("elements")):
-            yield rec | {"pivotValue": f"urn:li:{self.search_param_value}:{self.get_primary_key_from_slice(kwargs.get('stream_slice'))}"}
+            yield rec | {self.search_param_value: self.get_primary_key_from_slice(kwargs.get("stream_slice")), "pivot": self.pivot_by}
 
 
 class AdCampaignAnalytics(LinkedInAdsAnalyticsStream):
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_analytics.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_analytics.json
@@ -107,9 +107,6 @@
     "otherEngagements": {
       "type": ["null", "number"]
     },
-    "pivotValue": {
-      "type": ["null", "string"]
-    },
     "pivotValues": {
       "type": ["null", "array"],
       "items": {
@@ -299,6 +296,9 @@
     },
     "viralVideoViews": {
       "type": ["null", "number"]
+    },
+    "pivot": {
+      "type": ["null", "string"]
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json
@@ -10,117 +10,141 @@
         "start.month": 1,
         "start.year": 2021
       },
-      "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions"
+      "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,pivotValues"
     },
     {
       "campaign_id": 123,
+      "fields": "externalWebsiteConversions,externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,dateRange",
       "dateRange": {
-        "end.day": 31,
-        "end.month": 1,
-        "end.year": 2021,
         "start.day": 1,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange"
+        "start.year": 2021,
+        "end.day": 31,
+        "end.month": 1,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "postClickJobApplyClicks,postClickRegistrations,postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,dateRange,pivotValues",
       "dateRange": {
-        "end.day": 31,
-        "end.month": 1,
-        "end.year": 2021,
         "start.day": 1,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange"
+        "start.year": 2021,
+        "end.day": 31,
+        "end.month": 1,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "videoViews,viralCardClicks,viralCardImpressions,viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,dateRange,pivotValues",
       "dateRange": {
-        "end.day": 31,
-        "end.month": 1,
-        "end.year": 2021,
         "start.day": 1,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange"
+        "start.year": 2021,
+        "end.day": 31,
+        "end.month": 1,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,dateRange,pivotValues",
       "dateRange": {
+        "start.day": 1,
+        "start.month": 1,
+        "start.year": 2021,
         "end.day": 31,
         "end.month": 1,
-        "end.year": 2021,
+        "end.year": 2021
+      }
+    },
+    {
+      "campaign_id": 123,
+      "fields": "viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange,pivotValues",
+      "dateRange": {
         "start.day": 1,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange"
+        "start.year": 2021,
+        "end.day": 31,
+        "end.month": 1,
+        "end.year": 2021
+      }
     }
   ],
   [
     {
       "campaign_id": 123,
+      "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,pivotValues",
       "dateRange": {
-        "end.day": 2,
-        "end.month": 3,
-        "end.year": 2021,
         "start.day": 31,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions"
+        "start.year": 2021,
+        "end.day": 2,
+        "end.month": 3,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "externalWebsiteConversions,externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,dateRange",
       "dateRange": {
-        "end.day": 2,
-        "end.month": 3,
-        "end.year": 2021,
         "start.day": 31,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange"
+        "start.year": 2021,
+        "end.day": 2,
+        "end.month": 3,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "postClickJobApplyClicks,postClickRegistrations,postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,dateRange,pivotValues",
       "dateRange": {
-        "end.day": 2,
-        "end.month": 3,
-        "end.year": 2021,
         "start.day": 31,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange"
+        "start.year": 2021,
+        "end.day": 2,
+        "end.month": 3,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "videoViews,viralCardClicks,viralCardImpressions,viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,dateRange,pivotValues",
       "dateRange": {
-        "end.day": 2,
-        "end.month": 3,
-        "end.year": 2021,
         "start.day": 31,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange"
+        "start.year": 2021,
+        "end.day": 2,
+        "end.month": 3,
+        "end.year": 2021
+      }
     },
     {
       "campaign_id": 123,
+      "fields": "viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,dateRange,pivotValues",
       "dateRange": {
+        "start.day": 31,
+        "start.month": 1,
+        "start.year": 2021,
         "end.day": 2,
         "end.month": 3,
-        "end.year": 2021,
+        "end.year": 2021
+      }
+    },
+    {
+      "campaign_id": 123,
+      "fields": "viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange,pivotValues",
+      "dateRange": {
         "start.day": 31,
         "start.month": 1,
-        "start.year": 2021
-      },
-      "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange"
+        "start.year": 2021,
+        "end.day": 2,
+        "end.month": 3,
+        "end.year": 2021
+      }
     }
   ]
 ]
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json
@@ -22,6 +22,7 @@
           "year": 2023
         }
       },
+      "pivotValues": ["urn:li:sponsoredCreative:1"],
       "commentLikes": 0,
       "adUnitClicks": 0,
       "companyPageClicks": 0,
@@ -53,6 +54,7 @@
           "year": 2023
         }
       },
+      "pivotValues": ["urn:li:sponsoredCreative:1"],
       "commentLikes": 0,
       "adUnitClicks": 0,
       "companyPageClicks": 0,
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json
@@ -19,6 +19,7 @@
           "year": 2023
         }
       },
+      "pivotValues": ["urn:li:sponsoredCreative:1"],
       "viralCardImpressions": 0,
       "videoFirstQuartileCompletions": 0,
       "textUrlClicks": 0,
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py
@@ -81,10 +81,10 @@ def test_chunk_analytics_fields():
     with "dateRange" field presented in each chunk.
     """
     expected_output = [
-        ["field_1", "base_field_1", "field_2", "dateRange"],
-        ["base_field_2", "field_3", "field_4", "dateRange"],
-        ["field_5", "field_6", "field_7", "dateRange"],
-        ["field_8", "dateRange"],
+        ["field_1", "base_field_1", "field_2", "dateRange", "pivotValues"],
+        ["base_field_2", "field_3", "field_4", "dateRange", "pivotValues"],
+        ["field_5", "field_6", "field_7", "dateRange", "pivotValues"],
+        ["field_8", "dateRange", "pivotValues"],
     ]
 
     assert list(LinkedInAdsAnalyticsStream.chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output
diff --git a/docs/integrations/sources/linkedin-ads-migrations.md b/docs/integrations/sources/linkedin-ads-migrations.md
@@ -0,0 +1,40 @@
+# LinkedIn Ads Migration Guide
+
+## Upgrading to 1.0.0
+
+Version 1.0.0 introduces changes in primary key for all *-analytics streams (including custom ones).
+- "ad_campaign_analytics"
+- "ad_creative_analytics"
+- "ad_impression_device_analytics"
+- "ad_member_company_size_analytics"
+- "ad_member_country_analytics"
+- "ad_member_job_function_analytics"
+- "ad_member_job_title_analytics"
+- "ad_member_industry_analytics"
+- "ad_member_seniority_analytics"
+- "ad_member_region_analytics"
+- "ad_member_company_analytics"
+
+## Migration Steps
+
+### Refresh affected schemas and reset data
+
+1. Select **Connections** in the main nav bar.
+    1. Select the connection(s) affected by the update.
+2. Select the **Replication** tab.
+    1. Select **Refresh source schema**.
+    2. Select **OK**.
+:::note
+Any detected schema changes will be listed for your review.
+:::
+3. Select **Save changes** at the bottom of the page.
+    1. Ensure the **Reset affected streams** option is checked.
+:::note
+Depending on destination type you may not be prompted to reset your data.
+:::
+4. Select **Save connection**. 
+:::note
+This will reset the data in your destination and initiate a fresh sync.
+:::
+
+For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset).
diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md

Original file line number	Diff line number	Diff line change
`@@ -107,9 +107,6 @@`
`107`	`107`	`"otherEngagements": {`
`108`	`108`	`"type": ["null", "number"]`
`109`	`109`	`},`
`110`		`- "pivotValue": {`
`111`		`- "type": ["null", "string"]`
`112`		`- },`
`113`	`110`	`"pivotValues": {`
`114`	`111`	`"type": ["null", "array"],`
`115`	`112`	`"items": {`
`@@ -299,6 +296,9 @@`
`299`	`296`	`},`
`300`	`297`	`"viralVideoViews": {`
`301`	`298`	`"type": ["null", "number"]`
	`299`	`+ },`
	`300`	`+ "pivot": {`
	`301`	`+ "type": ["null", "string"]`
`302`	`302`	`}`
`303`	`303`	`}`
`304`	`304`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@`
`19`	`19`	`"year": 2023`
`20`	`20`	`}`
`21`	`21`	`},`
	`22`	`+ "pivotValues": ["urn:li:sponsoredCreative:1"],`
`22`	`23`	`"viralCardImpressions": 0,`
`23`	`24`	`"videoFirstQuartileCompletions": 0,`
`24`	`25`	`"textUrlClicks": 0,`