Source Azure Blob Storage: add integration tests

airbyte-integrations/connectors/source-azure-blob-storage/.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    source_azure_blob_storage/run.py

airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dagger import Container
+
+
+async def pre_connector_install(base_image_container: Container) -> Container:
+    """
+    Docker compose is required to run the integration tests so we install Docker on top of the base image.
+    """
+    return (
+        base_image_container.with_exec(["sh", "-c", "apt-get update && apt-get install -y curl jq"])
+        # Download install-docker.sh script
+        .with_exec(["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/install-docker.sh"])
+        # Run the install-docker.sh script with a pinned Docker version
+        .with_exec(["sh", "/tmp/install-docker.sh", "--version", "25.0"])
+        # Remove the install-docker.sh script
+        .with_exec(["rm", "/tmp/install-docker.sh"])
+    )

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_avro.json

@@ -0,0 +1,35 @@
+{
+  "azure_blob_storage_endpoint": "http://localhost:10000/account1",
+  "azure_blob_storage_account_name": "account1",
+  "azure_blob_storage_account_key": "key1",
+  "azure_blob_storage_container_name": "testcontainer",
+  "streams": [
+    {
+      "name": "users",
+      "file_type": "avro",
+      "globs": ["**/test_avro_users*.avro"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "avro"
+      }
+    },
+    {
+      "name": "purchases",
+      "file_type": "avro",
+      "globs": ["**/test_avro_purchases*.avro"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "avro"
+      }
+    },
+    {
+      "name": "products",
+      "file_type": "avro",
+      "globs": ["**/test_avro_products*.avro"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "avro"
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_csv.json

@@ -0,0 +1,128 @@
+{
+  "azure_blob_storage_endpoint": "http://localhost:10000/account1",
+  "azure_blob_storage_account_name": "account1",
+  "azure_blob_storage_account_key": "key1",
+  "azure_blob_storage_container_name": "testcontainer",
+  "streams": [
+    {
+      "name": "users",
+      "file_type": "csv",
+      "globs": ["**/test_csv_users*.csv"],
+      "legacy_prefix": "",
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "csv",
+        "delimiter": ",",
+        "quote_char": "\"",
+        "double_quote": true,
+        "null_values": [
+          "",
+          "#N/A",
+          "#N/A N/A",
+          "#NA",
+          "-1.#IND",
+          "-1.#QNAN",
+          "-NaN",
+          "-nan",
+          "1.#IND",
+          "1.#QNAN",
+          "N/A",
+          "NA",
+          "NULL",
+          "NaN",
+          "n/a",
+          "nan",
+          "null"
+        ],
+        "true_values": ["1", "True", "TRUE", "true"],
+        "false_values": ["0", "False", "FALSE", "false"],
+        "inference_type": "Primitive Types Only",
+        "strings_can_be_null": false,
+        "encoding": "utf8",
+        "header_definition": {
+          "header_definition_type": "From CSV"
+        }
+      }
+    },
+    {
+      "name": "purchases",
+      "file_type": "csv",
+      "globs": ["**/test_csv_purchases*.csv"],
+      "legacy_prefix": "",
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "csv",
+        "delimiter": ",",
+        "quote_char": "\"",
+        "double_quote": true,
+        "null_values": [
+          "",
+          "#N/A",
+          "#N/A N/A",
+          "#NA",
+          "-1.#IND",
+          "-1.#QNAN",
+          "-NaN",
+          "-nan",
+          "1.#IND",
+          "1.#QNAN",
+          "N/A",
+          "NA",
+          "NULL",
+          "NaN",
+          "n/a",
+          "nan",
+          "null"
+        ],
+        "true_values": ["1", "True", "TRUE", "true"],
+        "false_values": ["0", "False", "FALSE", "false"],
+        "inference_type": "Primitive Types Only",
+        "strings_can_be_null": false,
+        "encoding": "utf8",
+        "header_definition": {
+          "header_definition_type": "From CSV"
+        }
+      }
+    },
+    {
+      "name": "products",
+      "file_type": "csv",
+      "globs": ["**/test_csv_products*.csv"],
+      "legacy_prefix": "",
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "csv",
+        "delimiter": ",",
+        "quote_char": "\"",
+        "double_quote": true,
+        "null_values": [
+          "",
+          "#N/A",
+          "#N/A N/A",
+          "#NA",
+          "-1.#IND",
+          "-1.#QNAN",
+          "-NaN",
+          "-nan",
+          "1.#IND",
+          "1.#QNAN",
+          "N/A",
+          "NA",
+          "NULL",
+          "NaN",
+          "n/a",
+          "nan",
+          "null"
+        ],
+        "true_values": ["1", "True", "TRUE", "true"],
+        "false_values": ["0", "False", "FALSE", "false"],
+        "inference_type": "Primitive Types Only",
+        "strings_can_be_null": false,
+        "encoding": "utf8",
+        "header_definition": {
+          "header_definition_type": "From CSV"
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_jsonl.json

@@ -0,0 +1,41 @@
+{
+  "azure_blob_storage_endpoint": "http://localhost:10000/account1",
+  "azure_blob_storage_account_name": "account1",
+  "azure_blob_storage_account_key": "key1",
+  "azure_blob_storage_container_name": "testcontainer",
+  "streams": [
+    {
+      "name": "users",
+      "file_type": "jsonl",
+      "globs": ["**/test_jsonl_users*.jsonl"],
+      "legacy_prefix": "",
+      "newlines_in_values": true,
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "jsonl"
+      }
+    },
+    {
+      "name": "purchases",
+      "file_type": "jsonl",
+      "globs": ["**/test_jsonl_purchases*.jsonl"],
+      "legacy_prefix": "",
+      "newlines_in_values": true,
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "jsonl"
+      }
+    },
+    {
+      "name": "products",
+      "file_type": "jsonl",
+      "globs": ["**/test_jsonl_products*.jsonl"],
+      "legacy_prefix": "",
+      "newlines_in_values": true,
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "jsonl"
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_parquet.json

@@ -0,0 +1,35 @@
+{
+  "azure_blob_storage_endpoint": "http://localhost:10000/account1",
+  "azure_blob_storage_account_name": "account1",
+  "azure_blob_storage_account_key": "key1",
+  "azure_blob_storage_container_name": "testcontainer",
+  "streams": [
+    {
+      "name": "users",
+      "file_type": "jsonl",
+      "globs": ["**/test_parquet_users*.parquet"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "parquet"
+      }
+    },
+    {
+      "name": "purchases",
+      "file_type": "jsonl",
+      "globs": ["**/test_parquet_purchases*.parquet"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "parquet"
+      }
+    },
+    {
+      "name": "products",
+      "file_type": "jsonl",
+      "globs": ["**/test_parquet_products*.parquet"],
+      "validation_policy": "Emit Record",
+      "format": {
+        "filetype": "parquet"
+      }
+    }
+  ]
+}