🎉 Source Shopify: migrate products, product images and product variants to GraphQL BULK (#37767)

Author: bazarnov

airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml

@@ -25,8 +25,9 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         backward_compatibility_tests_config:
-          # The cursor field for `fulfillments` stream has changed from `id` to `updated_at`
-          disable_for_version: "1.1.8"
+          # the `product_variants` steam schema has changed, mainly:
+          # see this PR: https://github.com/airbytehq/airbyte/pull/37767
+          disable_for_version: "2.0.8"
   basic_read:
     tests:
       - config_path: "secrets/config_transactions_with_user_id.json"

airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json

@@ -420,13 +420,7 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "id": 99999999999999,
-        "products": {
-          "updated_at": "2027-07-11T13:07:45-07:00",
-          "deleted": {
-            "deleted_at": "2027-07-11T13:07:45-07:00"
-          }
-        }
+        "updated_at": "2027-07-11T13:07:45-07:00"
       },
       "stream_descriptor": {
         "name": "product_images"
@@ -454,13 +448,6 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "id": 99999999999999,
-        "products": {
-          "updated_at": "2027-07-11T13:07:45-07:00",
-          "deleted": {
-            "deleted_at": "2027-07-11T13:07:45-07:00"
-          }
-        },
         "updated_at": "2027-07-11T13:07:45-07:00"
       },
       "stream_descriptor": {

airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl

@@ -70,9 +70,8 @@
 {"stream": "price_rules", "data": {"id": 1112171741373, "value_type": "fixed_amount", "value": "-10.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": false, "usage_limit": null, "starts_at": "2017-01-19T09:59:10-08:00", "ends_at": null, "created_at": "2022-10-14T10:19:39-07:00", "updated_at": "2023-04-14T05:24:53-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "New Title 2023", "admin_graphql_api_id": "gid://shopify/PriceRule/1112171741373", "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953899511}
 {"stream": "price_rules", "data": {"id": 945000284349, "value_type": "percentage", "value": "-3.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": true, "usage_limit": 10, "starts_at": "2021-07-07T07:22:04-07:00", "ends_at": null, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "1V8Z165KSH5T", "admin_graphql_api_id": "gid://shopify/PriceRule/945000284349", "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953899512}
 {"stream": "price_rules", "data": {"id": 945205379261, "value_type": "percentage", "value": "-100.0", "customer_selection": "all", "target_type": "shipping_line", "target_selection": "all", "allocation_method": "each", "allocation_limit": null, "once_per_customer": false, "usage_limit": null, "starts_at": "2021-07-08T05:40:13-07:00", "ends_at": "2024-01-01T23:59:59-08:00", "created_at": "2021-07-08T05:40:37-07:00", "updated_at": "2023-12-07T03:40:44-08:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": {"greater_than_or_equal_to": "1.0"}, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "HZAVNV2487WC", "admin_graphql_api_id": "gid://shopify/PriceRule/945205379261", "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953899513}
-{"stream": "product_images", "data": {"id": 29301295481021, "alt": null, "position": 1, "product_id": 6796218138813, "created_at": "2021-06-22T18:09:28-07:00", "updated_at": "2021-06-22T18:09:28-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295481021", "width": 4393, "height": 2929, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/tin-of-beard-balm.jpg?v=1624410568", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953901149}
-{"stream": "product_images", "data": {"id": 29301295513789, "alt": null, "position": 1, "product_id": 6796218269885, "created_at": "2021-06-22T18:09:29-07:00", "updated_at": "2021-06-22T18:09:29-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295513789", "width": 3840, "height": 2560, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/pair-of-all-black-sneakers.jpg?v=1624410569", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953901154}
-{"stream": "product_images", "data": {"id": 29301295546557, "alt": null, "position": 1, "product_id": 6796218302653, "created_at": "2021-06-22T18:09:29-07:00", "updated_at": "2021-06-22T18:09:29-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295546557", "width": 3960, "height": 2640, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/red-silver-fishing-lure.jpg?v=1624410569", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953901155}
+{"stream": "product_images", "data": {"created_at": "2023-04-14T10:34:46+00:00", "updated_at": "2023-04-14T11:05:13+00:00", "id": 33290489659581, "height": 64, "alt": "Test", "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/Airbytelogo64x64.png?v=1681468487", "width": 64, "admin_graphql_api_id": "gid://shopify/ProductImage/33290489659581", "product_id": 6796229574845, "shop_url": "airbyte-integration-test"}, "emitted_at": 1714673982582}
+{"stream": "product_images", "data": {"created_at": "2021-06-23T01:09:47+00:00", "updated_at": "2023-04-24T17:27:15+00:00", "id": 29301297316029, "height": 1467, "alt": "updated_mon_24.04.2023", "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1624410587", "width": 2200, "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029", "product_id": 6796220989629, "shop_url": "airbyte-integration-test"}, "emitted_at": 1714673982587}
 {"stream": "products", "data": {"id": 6796217909437, "title": "Red And Navy Tee Sleeve", "body_html": "Zoom in on the sleeve of a red t-shirt with navy blue trim along the sleeve. Looks like a great tennis outfit.", "vendor": "Little Group", "product_type": "Movies", "created_at": "2021-06-22T18:09:27-07:00", "handle": "red-and-navy-tee-sleeve", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217909437", "variants": [{"id": 40090579992765, "product_id": 6796217909437, "title": "Plastic", "price": 23.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Plastic", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 39, "weight": 39.0, "weight_unit": "g", "inventory_item_id": 42185194700989, "inventory_quantity": 3, "old_inventory_quantity": 3, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090579992765", "image_id": null}], "options": [{"id": 8720175235261, "product_id": 6796217909437, "name": "Title", "position": 1, "values": ["Plastic"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953903012}
 {"stream": "products", "data": {"id": 6796217942205, "title": "Grey T-Shirt", "body_html": "A grey t-shirt on a hanger. Simple. Classic. Grey.", "vendor": "Lang - Bogisich", "product_type": "Home", "created_at": "2021-06-22T18:09:27-07:00", "handle": "grey-t-shirt", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217942205", "variants": [{"id": 40090580025533, "product_id": 6796217942205, "title": "Granite", "price": 70.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Granite", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 0, "weight": 0.0, "weight_unit": "g", "inventory_item_id": 42185194733757, "inventory_quantity": 38, "old_inventory_quantity": 38, "requires_shipping": false, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090580025533", "image_id": null}], "options": [{"id": 8720175268029, "product_id": 6796217942205, "name": "Title", "position": 1, "values": ["Granite"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953903015}
 {"stream": "products", "data": {"id": 6796217974973, "title": "Pool Floaty Icecream", "body_html": "Inflatable pink ice cream pool toy.", "vendor": "Fritsch - Ferry", "product_type": "Grocery", "created_at": "2021-06-22T18:09:27-07:00", "handle": "pool-floaty-icecream", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217974973", "variants": [{"id": 40090580091069, "product_id": 6796217974973, "title": "magenta", "price": 57.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "magenta", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 499, "weight": 499.0, "weight_unit": "g", "inventory_item_id": 42185194766525, "inventory_quantity": 1, "old_inventory_quantity": 1, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090580091069", "image_id": null}], "options": [{"id": 8720175300797, "product_id": 6796217974973, "name": "Title", "position": 1, "values": ["magenta"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1708953903015}