📝 🎉 Source Dixa: Migrated to CI Sandbox, refactored code structure for future support (#5367)

bazarnov · web-flow · commit 0f55bf63d555 · 2021-08-18T21:59:33.000+03:00
diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -88,6 +88,7 @@ jobs:
           DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
           DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
           DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
+          SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
           EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
           FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}
diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -88,6 +88,7 @@ jobs:
           DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
           DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
           DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
+          SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
           EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
           FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}
diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/0b5c867e-1b12-4d02-ab74-97b2184ff6d7.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/0b5c867e-1b12-4d02-ab74-97b2184ff6d7.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7",
   "name": "Dixa",
   "dockerRepository": "airbyte/source-dixa",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -385,7 +385,7 @@
 - sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7
   name: Dixa
   dockerRepository: airbyte/source-dixa
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/dixa
 - sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474
   name: Typeform
diff --git a/airbyte-integrations/connectors/source-dixa/Dockerfile b/airbyte-integrations/connectors/source-dixa/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-dixa
diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml
@@ -14,20 +14,13 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
-#      expect_records:
-#        path: "integration_tests/expected_records.txt"
-#        extra_fields: no
-#        exact_order: no
-#        extra_records: yes
+      timeout_seconds: 1200
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-#      future_state_path: "integration_tests/abnormal_state.json"
-# We skip the full_refresh test because of unusual behaviour in the Dixa API.
-# We observed cases where a record was updated without the updated_at value changing.
-# See the thread below for further information:
-# https://airbytehq.slack.com/archives/C01VDDEGL7M/p1625319909273300
-#  full_refresh:
-#    - config_path: "secrets/config.json"
-#      configured_catalog_path: "integration_tests/configured_catalog.json"
+      future_state_path: "integration_tests/abnormal_state.json"
+      timeout_seconds: 1200
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      timeout_seconds: 1200
diff --git a/airbyte-integrations/connectors/source-dixa/build.gradle b/airbyte-integrations/connectors/source-dixa/build.gradle
@@ -1,8 +1,7 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    // TODO acceptance tests are disabled in CI pending a Dixa Sandbox: https://github.com/airbytehq/airbyte/issues/4667
-//    id 'airbyte-source-acceptance-test'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-dixa/integration_tests/abnormal_state.json
@@ -1,5 +1,5 @@
 {
   "conversation_export": {
-    "updated_at": 9999999999999
+    "updated_at": 4778168400000
   }
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py
@@ -30,7 +30,4 @@
 
 @pytest.fixture(scope="session", autouse=True)
 def connector_setup():
-    """ This fixture is a placeholder for external resources that acceptance test might require."""
-    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
     yield
-    # TODO: clean up test dependencies
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/catalog.json b/airbyte-integrations/connectors/source-dixa/integration_tests/catalog.json
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-dixa/integration_tests/configured_catalog.json
@@ -1,17 +1,17 @@
 {
   "streams": [
     {
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"],
       "stream": {
         "name": "conversation_export",
+        "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]],
-        "json_schema": {}
-      }
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["updated_at"],
+      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/invalid_config.json
@@ -1,4 +1,5 @@
 {
-  "start_timestamp": 1625263200000,
-  "batch_size": 1
+  "api_token": "TOKEN",
+  "start_date": "2020-01-01",
+  "batch_size": "31"
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json
@@ -1,3 +1,5 @@
 {
-  "fix-me": "TODO"
+  "api_token": "TOKEN",
+  "start_date": "2020-01-01",
+  "batch_size": "31"
 }
diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_state.json
@@ -1,5 +1,5 @@
 {
-  "todo-stream-name": {
-    "todo-field-name": "value"
+  "conversation_export": {
+    "updated_at": 1628748165633
   }
 }
diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json
@@ -1,18 +1,18 @@
 {
-  "type": "object",
+  "type": ["null", "object"],
   "required": ["id", "created_at", "initial_channel", "requester_id"],
   "properties": {
     "id": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "created_at": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "initial_channel": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "requester_id": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "requester_name": {
       "type": ["null", "string"]
@@ -42,9 +42,9 @@
       "type": ["null", "string"]
     },
     "ratings": {
-      "type": "array",
+      "type": ["null", "array"],
       "items": {
-        "type": "object",
+        "type": ["null", "object"],
         "properties": {
           "rating_score": {
             "type": ["null", "integer"]
@@ -142,13 +142,13 @@
     "tags": {
       "type": ["null", "array"],
       "items": {
-        "type": "string"
+        "type": ["null", "string"]
       }
     },
     "conversation_wrapup_notes": {
       "type": ["null", "array"],
       "items": {
-        "type": "string"
+        "type": ["null", "string"]
       }
     },
     "transferee_name": {
diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py
diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json
diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/utils.py b/airbyte-integrations/connectors/source-dixa/source_dixa/utils.py
diff --git a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py
diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md
diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,6 @@`
`2`	`2`	`"sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7",`
`3`	`3`	`"name": "Dixa",`
`4`	`4`	`"dockerRepository": "airbyte/source-dixa",`
`5`		`- "dockerImageTag": "0.1.0",`
	`5`	`+ "dockerImageTag": "0.1.1",`
`6`	`6`	`"documentationUrl": "https://docs.airbyte.io/integrations/sources/dixa"`
`7`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,7 @@`
`1`	`1`	`plugins {`
`2`	`2`	`id 'airbyte-python'`
`3`	`3`	`id 'airbyte-docker'`
`4`		`- // TODO acceptance tests are disabled in CI pending a Dixa Sandbox: https://github.com/airbytehq/airbyte/issues/4667`
`5`		`-// id 'airbyte-source-acceptance-test'`
	`4`	`+ id 'airbyte-source-acceptance-test'`
`6`	`5`	`}`
`7`	`6`
`8`	`7`	`airbytePython {`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`	`2`	`"conversation_export": {`
`3`		`- "updated_at": 9999999999999`
	`3`	`+ "updated_at": 4778168400000`
`4`	`4`	`}`
`5`	`5`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,17 +1,17 @@`
`1`	`1`	`{`
`2`	`2`	`"streams": [`
`3`	`3`	`{`
`4`		`- "sync_mode": "incremental",`
`5`		`- "destination_sync_mode": "append",`
`6`		`- "cursor_field": ["updated_at"],`
`7`	`4`	`"stream": {`
`8`	`5`	`"name": "conversation_export",`
	`6`	`+ "json_schema": {},`
`9`	`7`	`"supported_sync_modes": ["full_refresh", "incremental"],`
`10`	`8`	`"source_defined_cursor": true,`
`11`	`9`	`"default_cursor_field": ["updated_at"],`
`12`		`- "source_defined_primary_key": [["id"]],`
`13`		`- "json_schema": {}`
`14`		`- }`
	`10`	`+ "source_defined_primary_key": [["id"]]`
	`11`	`+ },`
	`12`	`+ "sync_mode": "incremental",`
	`13`	`+ "cursor_field": ["updated_at"],`
	`14`	`+ "destination_sync_mode": "append"`
`15`	`15`	`}`
`16`	`16`	`]`
`17`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`{`
`2`		`- "start_timestamp": 1625263200000,`
`3`		`- "batch_size": 1`
	`2`	`+ "api_token": "TOKEN",`
	`3`	`+ "start_date": "2020-01-01",`
	`4`	`+ "batch_size": "31"`
`4`	`5`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
`1`	`1`	`{`
`2`		`- "fix-me": "TODO"`
	`2`	`+ "api_token": "TOKEN",`
	`3`	`+ "start_date": "2020-01-01",`
	`4`	`+ "batch_size": "31"`
`3`	`5`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`		`- "todo-stream-name": {`
`3`		`- "todo-field-name": "value"`
	`2`	`+ "conversation_export": {`
	`3`	`+ "updated_at": 1628748165633`
`4`	`4`	`}`
`5`	`5`	`}`