Merge branch 'master' into leti/experiment-add-source-in-signup

* master: (69 commits)
  🪟 🐛 Fix wrong geography dropdown type #19021
  SAT: basic read on full catalog when `test_strictness_level == high` (#18937)
  Unhide DynamoDB destination (#18994)
  Fixed tests for destination connectors (#19007)
  🐛 Source Facebook Marketing: handle FacebookBadObjectError (#18971)
  Edit multi-cloud docs (#18972)
  🪟 🎉 Load credits consumption separate (#18986)
  Bmoric/extract source api (#18944)
  Migrating InvalidCursorException -> ConfigErrorException  (#18995)
  🪟 🎨 Fix banner link color (#18978)
  Handling configuration exceptions in IntegrationRunner (#18989)
  Add new workspace api endpoint (#18983)
  Add normalization to destination definition and actor definition table (#18300)
  Fix oauth controller (#18981)
  Fix migration dev center schema dump by run db-specific initialization script (#18984)
  fix master build failure (#18982)
  cleanup: delete debezium 1-4-2 module (#18733)
  Remove unused job persistence methods. (#18952)
  Hash filenames of extracted CSS (#18976)
  Fix typo in source code comment DataDaog ==> Datadog (#18911)
  ...

Author: letiescanciano

.github/actions/build-branch/action.yml

@@ -18,7 +18,7 @@ runs:
         # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
         #
         [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "branch_version_tag=${{ inputs.branch_version_tag }}" >> $GITHUB_OUTPUT \
-          || { short_hash=$(git rev-parse --short HEAD); echo "branch_version_tag=dev-$short_hash" >> $GITHUB_OUTPUT ; }
+          || { short_hash=$(git rev-parse --short=7 HEAD); echo "branch_version_tag=dev-$short_hash" >> $GITHUB_OUTPUT ; }
 
     - uses: actions/setup-java@v1
       with:

.github/comment_templates/connector_dependency_template.md

@@ -1,5 +1,7 @@
 <!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
 
+## Affected Connector Report
+
 NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do the following as needed:
 - Run integration tests
 - Bump connector version
@@ -17,6 +19,8 @@ NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do
 | --- | :---: | :---: | :---: |
 {source_rows}
 
+* See "Actionable Items" below for how to resolve warnings and errors.
+
 </details>
 
 <details {destination_open}>
@@ -30,6 +34,8 @@ NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do
 | --- | :---: | :---: | :---: |
 {destination_rows}
 
+* See "Actionable Items" below for how to resolve warnings and errors.
+
 </details>
 
 {others}
@@ -38,7 +44,9 @@ NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do
 
 <summary>
 
-### Notes
+### Actionable Items
+
+(click to expand)
 
 </summary>
 

.github/teams.yml

@@ -0,0 +1,3 @@
+team/growth:
+  - "@letiescanciano"
+  - "@arnaudjnn"

.github/workflows/deploy-oss-catalog.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths: 
-      - airbyte-config/init/src/main/resources/seed
+      - airbyte-config/init/src/main/resources/seed/**
 
   workflow_dispatch:
 

.github/workflows/label-pr-by-team.yml

@@ -0,0 +1,9 @@
+name: "Add labels to github PRs based on team"
+on: pull_request
+jobs:
+  team-labeler:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JulienKode/[email protected]
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/platform-workflow-labels.yml

@@ -11,19 +11,31 @@ jobs:
     permissions:
       issues: write
     steps:
-      - name: Label Platform Workflow Issue
+      - name: Label Issue
         uses: andymckay/[email protected]
         with:
           add-labels: "platform-workflow/requires-grooming"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Add Issue To Project
+        uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/airbytehq/projects/25
+          github-token: ${{ secrets.PLATFORM_WORKFLOW_PROJECT_AUTOMATION }}
   unlabel_issues:
     if: ${{ github.event.action == 'unlabeled' && github.event.label.name == 'team/platform-workflow' }}
     runs-on: ubuntu-latest
     permissions:
       issues: write
     steps:
-      - name: Unlabel Platform Workflow Issue
+      - name: Unlabel Issue
         uses: andymckay/[email protected]
         with:
           remove-labels: "platform-workflow/requires-grooming"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Remove Issue From Project
+        uses: monry/[email protected]
+        with:
+          github-token: ${{ secrets.PLATFORM_WORKFLOW_PROJECT_AUTOMATION }}
+          project-owner: airbytehq
+          project-number: 25
+          issue-id: ${{ github.event.issue.node_id }}

.github/workflows/publish-oss-for-cloud.yml

@@ -62,7 +62,7 @@ jobs:
         run: |-
           set -x
 
-          commit_sha=$(git rev-parse --short HEAD)
+          commit_sha=$(git rev-parse --short=7 HEAD)
 
           # set dev_tag
           # AirbyteVersion.java allows versions that have a prefix of 'dev'
@@ -72,6 +72,7 @@ jobs:
           if $(git merge-base --is-ancestor "${commit_sha}" master); then
             echo "master_tag=${commit_sha}" >> $GITHUB_OUTPUT
           fi
+          cat $GITHUB_OUTPUT || true # for the sake of investigation
 
   oss-branch-build:
     name: "Gradle Build and Publish"

.github/workflows/report-connectors-dependency.yml

@@ -25,7 +25,8 @@ jobs:
             echo "comment=true" >> $GITHUB_OUTPUT
           fi
       - name: Find existing comment for connector dependencies
-        if: steps.dependency_report.outputs.comment == 'true'
+        # Always run this step because the action may need to
+        # remove a comment created from a previous commit.
         uses: peter-evans/find-comment@v2
         id: find_comment
         with:
@@ -40,3 +41,14 @@ jobs:
           comment-id: ${{ steps.find_comment.outputs.comment-id }}
           edit-mode: "replace"
           body-file: "comment_body.md"
+      - name: Remove deprecated report in PR
+        if: steps.dependency_report.outputs.comment != 'true' && steps.find_comment.outputs.comment-id != ''
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.find_comment.outputs.comment-id }}
+          edit-mode: "replace"
+          body: |
+            <!--- this comment is for `report-connectors-dependency.yml` identification, do not remove -->
+            ## Affected Connector Report
+            The latest commit has removed all connector-related changes. There are no more dependent connectors for this PR.

airbyte-api/src/main/openapi/config.yaml

@@ -168,6 +168,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/workspaces/get_by_connection_id:
+    post:
+      tags:
+        - workspace
+      summary: Find workspace by connection id
+      operationId: getWorkspaceByConnectionId
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ConnectionIdRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkspaceRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/workspaces/update:
     post:
       tags:
@@ -2266,6 +2289,26 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/InternalOperationResult"
+  /v1/attempt/save_stats:
+    post:
+      tags:
+        - attempt
+        - internal
+      summary: For worker to set sync stats of a running attempt.
+      operationId: saveStats
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SaveStatsRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful Operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/InternalOperationResult"
 
 components:
   securitySchemes:
@@ -4049,6 +4092,12 @@ components:
         recordsCommitted:
           type: integer
           format: int64
+        estimatedRecords:
+          type: integer
+          format: int64
+        estimatedBytes:
+          type: integer
+          format: int64
     AttemptStreamStats:
       type: object
       required:
@@ -4892,6 +4941,23 @@ components:
         processingTaskQueue:
           type: string
           default: ""
+    SaveStatsRequestBody:
+      type: object
+      required:
+        - jobId
+        - attemptNumber
+        - stats
+      properties:
+        jobId:
+          $ref: "#/components/schemas/JobId"
+        attemptNumber:
+          $ref: "#/components/schemas/AttemptNumber"
+        stats:
+          $ref: "#/components/schemas/AttemptStats"
+        streamStats:
+          type: array
+          items:
+            $ref: "#/components/schemas/AttemptStreamStats"
     InternalOperationResult:
       type: object
       required:

airbyte-base-java-image/Dockerfile

@@ -6,7 +6,7 @@ WORKDIR /app
 
 RUN yum install -y tar
 
-# Add the DataDaog Java APM agent
+# Add the Datadog Java APM agent
 ADD https://dtdg.co/latest-java-tracer dd-java-agent.jar
 
 # Add the OpenTelemetry Java APM agent

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -148,10 +148,10 @@ public BootloaderApp(final Configs configs,
   public void load() throws Exception {
     LOGGER.info("Initializing databases...");
     DatabaseCheckFactory.createConfigsDatabaseInitializer(configsDslContext,
-        configs.getConfigsDatabaseInitializationTimeoutMs(), MoreResources.readResource(DatabaseConstants.CONFIGS_SCHEMA_PATH)).initialize();
+        configs.getConfigsDatabaseInitializationTimeoutMs(), MoreResources.readResource(DatabaseConstants.CONFIGS_INITIAL_SCHEMA_PATH)).initialize();
 
     DatabaseCheckFactory.createJobsDatabaseInitializer(jobsDslContext,
-        configs.getJobsDatabaseInitializationTimeoutMs(), MoreResources.readResource(DatabaseConstants.JOBS_SCHEMA_PATH)).initialize();
+        configs.getJobsDatabaseInitializationTimeoutMs(), MoreResources.readResource(DatabaseConstants.JOBS_INITIAL_SCHEMA_PATH)).initialize();
     LOGGER.info("Databases initialized.");
 
     LOGGER.info("Setting up config database and default workspace...");

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -141,7 +141,7 @@ void testBootloaderAppBlankDb() throws Exception {
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration
       // to show that you meant to make a new migration to the prod database
-      assertEquals("0.40.18.001", configsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.18.002", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
       assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.5.4
+Low-code: Get response.json in a safe way
+
+## 0.5.3
+Low-code: Replace EmptySchemaLoader with DefaultSchemaLoader to retain backwards compatibility
+Low-code: Evaluate backoff strategies at runtime
+
+## 0.5.2
+Low-code: Allow for read even when schemas are not defined for a connector yet
+
 ## 0.4.2
 Low-code: Fix off by one error with the stream slicers
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py

@@ -7,7 +7,7 @@
 
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
-from airbyte_cdk.sources.declarative.schema.json_schema import JsonSchema
+from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.declarative.types import Config, StreamSlice
@@ -48,7 +48,7 @@ class DeclarativeStream(Stream, JsonSchemaMixin):
     def __post_init__(self, options: Mapping[str, Any]):
         self.stream_cursor_field = self.stream_cursor_field or []
         self.transformations = self.transformations or []
-        self._schema_loader = self.schema_loader if self.schema_loader else JsonSchema(config=self.config, options=options)
+        self._schema_loader = self.schema_loader if self.schema_loader else DefaultSchemaLoader(config=self.config, options=options)
 
     @property
     def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:

airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py

@@ -19,4 +19,7 @@ class JsonDecoder(Decoder, JsonSchemaMixin):
     options: InitVar[Mapping[str, Any]]
 
     def decode(self, response: requests.Response) -> Union[Mapping[str, Any], List]:
-        return response.json() or {}
+        try:
+            return response.json()
+        except requests.exceptions.JSONDecodeError:
+            return {}

airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py

@@ -32,7 +32,7 @@
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.declarative.schema.json_schema import JsonSchema
+from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader
 from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer
 from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer
@@ -62,7 +62,8 @@
     "HttpRequester": HttpRequester,
     "InterpolatedBoolean": InterpolatedBoolean,
     "InterpolatedString": InterpolatedString,
-    "JsonSchema": JsonSchema,
+    "JsonSchema": JsonFileSchemaLoader,  # todo remove after hacktoberfest and update connectors to use JsonFileSchemaLoader
+    "JsonFileSchemaLoader": JsonFileSchemaLoader,
     "ListStreamSlicer": ListStreamSlicer,
     "MinMaxDatetime": MinMaxDatetime,
     "NoAuth": NoAuth,

airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py

@@ -30,7 +30,7 @@
 from airbyte_cdk.sources.declarative.requesters.requester import Requester
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
-from airbyte_cdk.sources.declarative.schema.json_schema import JsonSchema
+from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.stream_slicers.single_slice import SingleSlice
 from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer
@@ -58,7 +58,7 @@
     RequestOptionsProvider: InterpolatedRequestOptionsProvider,
     Requester: HttpRequester,
     Retriever: SimpleRetriever,
-    SchemaLoader: JsonSchema,
+    SchemaLoader: DefaultSchemaLoader,
     Stream: DeclarativeStream,
     StreamSlicer: SingleSlice,
 }

airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py

@@ -229,14 +229,17 @@ def _create_subcomponent(self, key, definition, kwargs, config, parent_class, in
                 options = kwargs.get(OPTIONS_STR, {})
                 try:
                     # enums can't accept options
-                    if issubclass(expected_type, enum.Enum):
+                    if issubclass(expected_type, enum.Enum) or self.is_primitive(definition):
                         return expected_type(definition)
                     else:
                         return expected_type(definition, options=options)
                 except Exception as e:
                     raise Exception(f"failed to instantiate type {expected_type}. {e}")
         return definition
 
+    def is_primitive(self, obj):
+        return isinstance(obj, (int, float, bool))
+
     @staticmethod
     def is_object_definition_with_class_name(definition):
         return isinstance(definition, dict) and "class_name" in definition