Merge branch 'master' into ddavydov/#2344-source-google-ads-refactor-date-slicing

Author: davydov-d

.github/workflows/connectors_tests.yml

@@ -11,7 +11,7 @@ on:
         default: "--modified"
       runner:
         description: "The runner to use for this job"
-        default: "conn-prod-xlarge-runner"
+        default: "dev-large-runner"
   pull_request:
     types:
       - opened
@@ -21,7 +21,7 @@ jobs:
   connectors_ci:
     name: Connectors CI
     timeout-minutes: 1440 # 24 hours
-    runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}}
+    runs-on: ${{ inputs.runner || 'dev-large-runner'}}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/publish_connectors.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   publish_connectors:
     name: Publish connectors
-    runs-on: conn-prod-xlarge-runner
+    runs-on: large-runner
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/report-connectors-dependency.yml

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.43.1
+current_version = 0.43.2
 commit = False
 
 [bumpversion:file:setup.py]

airbyte-cdk/python/.bumpversion.cfg

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.43.2
+Connector builder module: serialize request body as string
+
 ## 0.43.1
 Fix availability check to handle HttpErrors which happen during slice extraction
 

airbyte-cdk/python/CHANGELOG.md

@@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \
     && apk --no-cache add tzdata build-base
 
 # install airbyte-cdk
-RUN pip install --prefix=/install airbyte-cdk==0.43.1
+RUN pip install --prefix=/install airbyte-cdk==0.43.2
 
 # build a clean environment
 FROM base
@@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 # needs to be the same as CDK
-LABEL io.airbyte.version=0.43.1
+LABEL io.airbyte.version=0.43.2
 LABEL io.airbyte.name=airbyte/source-declarative-manifest

airbyte-cdk/python/Dockerfile

@@ -17,9 +17,9 @@ class HttpResponse:
 class HttpRequest:
     url: str
     parameters: Optional[Dict[str, Any]]
-    body: Optional[Dict[str, Any]]
     headers: Optional[Dict[str, Any]]
     http_method: str
+    body: Optional[str] = None
 
 
 @dataclass

airbyte-cdk/python/airbyte_cdk/connector_builder/models.py

@@ -5,7 +5,6 @@
 import json
 from dataclasses import InitVar, dataclass, field
 from itertools import islice
-from json import JSONDecodeError
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 import requests
@@ -496,22 +495,14 @@ def _prepared_request_to_airbyte_message(request: requests.PreparedRequest) -> A
         "url": request.url,
         "http_method": request.method,
         "headers": dict(request.headers),
-        "body": _body_binary_string_to_dict(request.body),
+        "body": _normalize_body_string(request.body),
     }
     log_message = filter_secrets(f"request:{json.dumps(request_dict)}")
     return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=log_message))
 
 
-def _body_binary_string_to_dict(body_str: str) -> Optional[Mapping[str, str]]:
-    if body_str:
-        if isinstance(body_str, (bytes, bytearray)):
-            body_str = body_str.decode()
-        try:
-            return json.loads(body_str)
-        except JSONDecodeError:
-            return {k: v for k, v in [s.split("=") for s in body_str.split("&")]}
-    else:
-        return None
+def _normalize_body_string(body_str: Optional[Union[str, bytes]]) -> Optional[str]:
+    return body_str.decode() if isinstance(body_str, (bytes, bytearray)) else body_str
 
 
 def _response_to_airbyte_message(response: requests.Response) -> AirbyteMessage:

airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -17,7 +17,7 @@
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.43.1",
+    version="0.43.2",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/setup.py

@@ -574,7 +574,7 @@ def test_path(test_name, requester_path, paginator_path, expected_path):
                 type=Type.LOG,
                 log=AirbyteLogMessage(
                     level=Level.INFO,
-                    message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/json", "Content-Length": "24"}, "body": {"b1": "v1", "b2": "v2"}}',
+                    message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/json", "Content-Length": "24"}, "body": "{\\"b1\\": \\"v1\\", \\"b2\\": \\"v2\\"}"}',
                 ),
             ),
         ),
@@ -590,23 +590,23 @@ def test_path(test_name, requester_path, paginator_path, expected_path):
                 type=Type.LOG,
                 log=AirbyteLogMessage(
                     level=Level.INFO,
-                    message='request:{"url": "https://airbyte.io/?p1=v1&p2=v2", "http_method": "GET", "headers": {"Content-Type": "application/json", "h1": "v1", "Content-Length": "24"}, "body": {"b1": "v1", "b2": "v2"}}',
+                    message='request:{"url": "https://airbyte.io/?p1=v1&p2=v2", "http_method": "GET", "headers": {"Content-Type": "application/json", "h1": "v1", "Content-Length": "24"}, "body": "{\\"b1\\": \\"v1\\", \\"b2\\": \\"v2\\"}"}',
                 ),
             ),
         ),
         (
             "test_get_request_with_request_body_data",
             HttpMethod.GET,
             "https://airbyte.io",
-            {"Content-Type": "application/json"},
+            {"Content-Type": "application/x-www-form-urlencoded"},
             {},
             {},
             {"b1": "v1", "b2": "v2"},
             AirbyteMessage(
                 type=Type.LOG,
                 log=AirbyteLogMessage(
                     level=Level.INFO,
-                    message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/json", "Content-Length": "11"}, "body": {"b1": "v1", "b2": "v2"}}',
+                    message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/x-www-form-urlencoded", "Content-Length": "11"}, "body": "b1=v1&b2=v2"}',
                 ),
             ),
         ),

airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py

@@ -24,5 +24,5 @@ ENV APPLICATION destination-gcs
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/destination-gcs

airbyte-integrations/connectors/destination-gcs/Dockerfile

@@ -2,11 +2,11 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: ca8f6566-e555-4b40-943a-545bf123117a
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.4.0
   dockerRepository: airbyte/destination-gcs
   githubIssueLabel: destination-gcs
   icon: googlecloudstorage.svg
-  license: MIT
+  license: ELv2
   name: Google Cloud Storage (GCS)
   registries:
     cloud:

airbyte-integrations/connectors/destination-gcs/metadata.yaml

@@ -49,5 +49,5 @@ RUN /bin/bash -c 'set -e && \
     fi'
 
 RUN yum clean all
-LABEL io.airbyte.version=0.4.2
+LABEL io.airbyte.version=0.5.0
 LABEL io.airbyte.name=airbyte/destination-s3

airbyte-integrations/connectors/destination-s3/Dockerfile

@@ -2,11 +2,11 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
-  dockerImageTag: 0.4.2
+  dockerImageTag: 0.5.0
   dockerRepository: airbyte/destination-s3
   githubIssueLabel: destination-s3
   icon: s3.svg
-  license: MIT
+  license: ELv2
   name: S3
   registries:
     cloud:

airbyte-integrations/connectors/destination-s3/metadata.yaml

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.4.3
+LABEL io.airbyte.version=0.5.0
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -5,11 +5,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 0.4.3
+  dockerImageTag: 0.5.0
   dockerRepository: airbyte/source-facebook-marketing
   githubIssueLabel: source-facebook-marketing
   icon: facebook.svg
-  license: MIT
+  license: ELv2
   name: Facebook Marketing
   registries:
     cloud:

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -34,5 +34,5 @@ COPY source_facebook_pages ./source_facebook_pages
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.5
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-facebook-pages

airbyte-integrations/connectors/source-facebook-pages/Dockerfile

@@ -5,11 +5,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 010eb12f-837b-4685-892d-0a39f76a98f5
-  dockerImageTag: 0.2.5
+  dockerImageTag: 0.3.0
   dockerRepository: airbyte/source-facebook-pages
   githubIssueLabel: source-facebook-pages
   icon: facebook.svg
-  license: MIT
+  license: ELv2
   name: Facebook Pages
   registries:
     cloud:

airbyte-integrations/connectors/source-facebook-pages/metadata.yaml

@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-gcs

airbyte-integrations/connectors/source-gcs/Dockerfile

@@ -2,11 +2,11 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.2.0
   dockerRepository: airbyte/source-gcs
   githubIssueLabel: source-gcs
   icon: gcs.svg
-  license: MIT
+  license: ELv2
   name: GCS
   releaseStage: alpha
   registries:

airbyte-integrations/connectors/source-gcs/metadata.yaml

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.25
+LABEL io.airbyte.version=0.3.1
 LABEL io.airbyte.name=airbyte/source-google-ads

airbyte-integrations/connectors/source-google-ads/Dockerfile

@@ -6,11 +6,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
-  dockerImageTag: 0.2.25
+  dockerImageTag: 0.3.1
   dockerRepository: airbyte/source-google-ads
   githubIssueLabel: source-google-ads
   icon: google-adwords.svg
-  license: MIT
+  license: Elv2
   name: Google Ads
   registries:
     cloud:

airbyte-integrations/connectors/source-google-ads/metadata.yaml

@@ -28,5 +28,5 @@ COPY source_google_analytics_data_api ./source_google_analytics_data_api
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.0.0
+LABEL io.airbyte.version=1.1.0
 LABEL io.airbyte.name=airbyte/source-google-analytics-data-api

airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile

@@ -7,11 +7,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
-  dockerImageTag: 1.0.0
+  dockerImageTag: 1.1.0
   dockerRepository: airbyte/source-google-analytics-data-api
   githubIssueLabel: source-google-analytics-data-api
   icon: google-analytics.svg
-  license: MIT
+  license: Elv2
   name: Google Analytics 4 (GA4)
   registries:
     cloud:

airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.36
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-google-analytics-v4

airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile

@@ -8,11 +8,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003
-  dockerImageTag: 0.1.36
+  dockerImageTag: 0.2.0
   dockerRepository: airbyte/source-google-analytics-v4
   githubIssueLabel: source-google-analytics-v4
   icon: google-analytics.svg
-  license: MIT
+  license: Elv2
   name: Google Analytics (Universal Analytics)
   registries:
     cloud:

airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.0.2
+LABEL io.airbyte.version=1.1.0
 LABEL io.airbyte.name=airbyte/source-google-search-console

airbyte-integrations/connectors/source-google-search-console/Dockerfile

@@ -5,11 +5,11 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8
-  dockerImageTag: 1.0.2
+  dockerImageTag: 1.1.0
   dockerRepository: airbyte/source-google-search-console
   githubIssueLabel: source-google-search-console
   icon: googlesearchconsole.svg
-  license: MIT
+  license: Elv2
   name: Google Search Console
   registries:
     cloud:

airbyte-integrations/connectors/source-google-search-console/metadata.yaml

@@ -34,5 +34,5 @@ COPY source_google_sheets ./source_google_sheets
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.39
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-google-sheets