🚨🚨 Source Linkedin Ads: Update pivotValues pk type for Ad Analytics streams (#37531)

Author: darynaishchenko

airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 137ece28-5434-455c-8f34-69dc3782f451
-  dockerImageTag: 1.0.1
+  dockerImageTag: 2.0.0
   dockerRepository: airbyte/source-linkedin-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads
   githubIssueLabel: source-linkedin-ads
@@ -48,6 +48,23 @@ data:
               - "ad_member_seniority_analytics"
               - "ad_member_region_analytics"
               - "ad_member_company_analytics"
+      2.0.0:
+        message: This upgrade changes primary key for *-analytics streams from pivotValues[array of strings] to string_of_pivot_values[string] so that it is compatible with more destination types.
+        upgradeDeadline: "2024-05-14"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes:
+              - "ad_campaign_analytics"
+              - "ad_creative_analytics"
+              - "ad_impression_device_analytics"
+              - "ad_member_company_size_analytics"
+              - "ad_member_country_analytics"
+              - "ad_member_job_function_analytics"
+              - "ad_member_job_title_analytics"
+              - "ad_member_industry_analytics"
+              - "ad_member_seniority_analytics"
+              - "ad_member_region_analytics"
+              - "ad_member_company_analytics"
   suggestedStreams:
     streams:
       - accounts

airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.0.1"
+version = "2.0.0"
 name = "source-linkedin-ads"
 description = "Source implementation for Linkedin Ads."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml

@@ -127,7 +127,7 @@ class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC):
 
     endpoint = "adAnalytics"
     # For Analytics streams, the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc.] + `end_date`
-    primary_key = ["pivotValues", "end_date"]
+    primary_key = ["string_of_pivot_values", "end_date"]
     cursor_field = "end_date"
     records_limit = 15000
     FIELDS_CHUNK_SIZE = 18

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py

@@ -113,6 +113,9 @@
         "type": ["null", "string"]
       }
     },
+    "string_of_pivot_values": {
+      "type": ["null", "string"]
+    },
     "postClickJobApplications": {
       "type": ["null", "number"]
     },

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_analytics.json

@@ -305,6 +305,12 @@ def transform_col_names(record: Dict, dict_keys: list = []) -> Mapping[str, Any]
     return record
 
 
+def transform_pivot_values(record: Dict) -> Mapping[str, Any]:
+    pivot_values = record.get("pivotValues", [])
+    record["string_of_pivot_values"] = ",".join(pivot_values)
+    return record
+
+
 def transform_data(records: List) -> Iterable[Mapping]:
     """
     We need to transform the nested complex data structures into simple key:value pair,
@@ -323,6 +329,9 @@ def transform_data(records: List) -> Iterable[Mapping]:
         if "variables" in record:
             record = transform_variables(record)
 
+        if "pivotValues" in record:
+            record = transform_pivot_values(record)
+
         record = transform_col_names(record, DESTINATION_RESERVED_KEYWORDS)
 
         yield record

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py

@@ -65,6 +65,7 @@
             }
         },
         "pivot": "TEST_PIVOT_VALUE",
+        "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"]
     }
 ]
 
@@ -142,5 +143,7 @@
         "start_date": "2021-08-13",
         "end_date": "2021-08-13",
         "_pivot": "TEST_PIVOT_VALUE",
+        "string_of_pivot_values": "TEST_PIVOT_VALUE_1,TEST_PIVOT_VALUE_2",
+        "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"]
     }
 ]

airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py

@@ -6,7 +6,7 @@
 from source_linkedin_ads.utils import transform_data
 
 
-def test_transfrom_data():
+def test_transform_data():
     """
     As far as we transform the data within the generator object,
     we use list() to have the actual output for the test assertion.

airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_transform_data.py

@@ -1,5 +1,36 @@
 # LinkedIn Ads Migration Guide
 
+## Upgrading to 2.0.0
+
+Version 2.0.0 introduces changes in the primary key selected for all *-analytics streams (including custom ones) from pivotValues[array of strings] to string_of_pivot_values[string] so that it is compatible with more destination types.
+
+- "ad_campaign_analytics"
+- "ad_creative_analytics"
+- "ad_impression_device_analytics"
+- "ad_member_company_size_analytics"
+- "ad_member_country_analytics"
+- "ad_member_job_function_analytics"
+- "ad_member_job_title_analytics"
+- "ad_member_industry_analytics"
+- "ad_member_seniority_analytics"
+- "ad_member_region_analytics"
+- "ad_member_company_analytics"
+
+## Migration Steps
+
+Clearing your data is required for the affected streams in order to continue syncing successfully. To clear your data for the affected streams, follow the steps below:
+
+1. Select **Connections** in the main navbar and select the connection(s) affected by the update.
+2. Select the **Schema** tab. 
+    1. Select **Refresh source schema** to bring in any schema changes. Any detected schema changes will be listed for your review.
+    2. Select **OK** to approve changes.
+3. Select **Save changes** at the bottom of the page. 
+    1. Ensure the **Clear affected streams** option is checked to ensure your streams continue syncing successfully with the new schema.
+4. Select **Save connection**. 
+
+This will clear the data in your destination for the subset of streams with schema changes. After the clear succeeds, trigger a sync by clicking **Sync Now**. For more information on clearing your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset).
+
+
 ## Upgrading to 1.0.0
 
 Version 1.0.0 introduces changes in the primary key selected for all *-analytics streams (including custom ones).

docs/integrations/sources/linkedin-ads-migrations.md

@@ -171,6 +171,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 2.0.0   | 2024-04-24 | [37531](https://github.com/airbytehq/airbyte/pull/37531) | Change primary key for Analytics Streams                                                                        |
 | 1.0.1   | 2024-03-28 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | Proceed pagination if return less than expected                                                                 |
 | 1.0.0   | 2024-04-10 | [36927](https://github.com/airbytehq/airbyte/pull/36927) | Update primary key for Analytics Streams                                                                        |
 | 0.8.0   | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0`                                                                                 |