refactor(source-sftp-bulk) bump CDK version to v4

airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configs/stream_excel_xls.json

@@ -0,0 +1,12 @@
+{
+  "streams": [
+    {
+      "name": "test_stream",
+      "file_type": "excel",
+      "globs": ["**/demo.xls"],
+      "format": {
+        "filetype": "excel"
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-sftp-bulk/integration_tests/configs/stream_excel_xlsx.json

@@ -0,0 +1,12 @@
+{
+  "streams": [
+    {
+      "name": "test_stream",
+      "file_type": "excel",
+      "globs": ["**/demo.xlsx", "**/demo1.xlsx"],
+      "format": {
+        "filetype": "excel"
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py

@@ -102,6 +102,16 @@ def config_fixture_password_all_jsonl(config) -> Mapping[str, Any]:
     yield config | load_config("stream_jsonl.json")
 
 
+@pytest.fixture(name="config_password_all_excel_xlsx", scope="session")
+def config_fixture_password_all_excel_xlsx(config) -> Mapping[str, Any]:
+    yield config | load_config("stream_excel_xlsx.json")
+
+
+@pytest.fixture(name="config_password_all_excel_xls", scope="session")
+def config_fixture_password_all_excel_xls(config) -> Mapping[str, Any]:
+    yield config | load_config("stream_excel_xls.json")
+
+
 @pytest.fixture(name="configured_catalog")
 def configured_catalog_fixture() -> ConfiguredAirbyteCatalog:
     stream_schema = {

airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py

@@ -59,6 +59,18 @@ def test_get_files_pattern_json_new_separator(configured_catalog: ConfiguredAirb
     assert len(output.records) == 3
 
 
+def test_get_all_files_excel_xlsx(configured_catalog: ConfiguredAirbyteCatalog, config_password_all_excel_xlsx: Mapping[str, Any]):
+    source = SourceSFTPBulk(catalog=configured_catalog, config=config_password_all_excel_xlsx, state=None)
+    output = read(source=source, config=config_password_all_excel_xlsx, catalog=configured_catalog)
+    assert len(output.records) == 2
+
+
+def test_get_all_files_excel_xls(configured_catalog: ConfiguredAirbyteCatalog, config_password_all_excel_xls: Mapping[str, Any]):
+    source = SourceSFTPBulk(catalog=configured_catalog, config=config_password_all_excel_xls, state=None)
+    output = read(source=source, config=config_password_all_excel_xls, catalog=configured_catalog)
+    assert len(output.records) == 1
+
+
 def test_get_files_pattern_no_match_json(configured_catalog: ConfiguredAirbyteCatalog, config_password_all_jsonl: Mapping[str, Any]):
     config_with_wrong_glob_pattern = deepcopy(config_password_all_jsonl)
     config_with_wrong_glob_pattern["streams"][0]["globs"] = ["**/not_existed_file.jsonl"]

airbyte-integrations/connectors/source-sftp-bulk/integration_tests/spec.json

@@ -285,7 +285,7 @@
                   "required": ["filetype"]
                 },
                 {
-                  "title": "Document File Type Format (Experimental)",
+                  "title": "Unstructured Document Format",
                   "type": "object",
                   "properties": {
                     "filetype": {
@@ -332,72 +332,25 @@
                           },
                           "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.",
                           "required": ["mode"]
-                        },
-                        {
-                          "title": "via API",
-                          "type": "object",
-                          "properties": {
-                            "mode": {
-                              "title": "Mode",
-                              "default": "api",
-                              "const": "api",
-                              "enum": ["api"],
-                              "type": "string"
-                            },
-                            "api_key": {
-                              "title": "API Key",
-                              "description": "The API key to use matching the environment",
-                              "default": "",
-                              "always_show": true,
-                              "airbyte_secret": true,
-                              "type": "string"
-                            },
-                            "api_url": {
-                              "title": "API URL",
-                              "description": "The URL of the unstructured API to use",
-                              "default": "https://api.unstructured.io",
-                              "always_show": true,
-                              "examples": ["https://api.unstructured.com"],
-                              "type": "string"
-                            },
-                            "parameters": {
-                              "title": "Additional URL Parameters",
-                              "description": "List of parameters send to the API",
-                              "default": [],
-                              "always_show": true,
-                              "type": "array",
-                              "items": {
-                                "title": "APIParameterConfigModel",
-                                "type": "object",
-                                "properties": {
-                                  "name": {
-                                    "title": "Parameter name",
-                                    "description": "The name of the unstructured API parameter to use",
-                                    "examples": [
-                                      "combine_under_n_chars",
-                                      "languages"
-                                    ],
-                                    "type": "string"
-                                  },
-                                  "value": {
-                                    "title": "Value",
-                                    "description": "The value of the parameter",
-                                    "examples": ["true", "hi_res"],
-                                    "type": "string"
-                                  }
-                                },
-                                "required": ["name", "value"]
-                              }
-                            }
-                          },
-                          "description": "Process files via an API, using the `hi_res` mode. This option is useful for increased performance and accuracy, but requires an API key and a hosted instance of unstructured.",
-                          "required": ["mode"]
                         }
                       ]
                     }
                   },
                   "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.",
                   "required": ["filetype"]
+                },
+                {
+                  "title": "Excel Format",
+                  "type": "object",
+                  "properties": {
+                    "filetype": {
+                      "title": "Filetype",
+                      "default": "excel",
+                      "const": "excel",
+                      "type": "string"
+                    }
+                  },
+                  "required": ["filetype"]
                 }
               ]
             },
@@ -406,6 +359,12 @@
               "description": "When enabled, syncs will not validate or structure records against the stream's schema.",
               "default": false,
               "type": "boolean"
+            },
+            "recent_n_files_to_read_for_schema_discovery": {
+              "title": "Files To Read For Schema Discover",
+              "description": "The number of resent files which will be used to discover the schema for this stream.",
+              "exclusiveMinimum": 0,
+              "type": "integer"
             }
           },
           "required": ["name", "format"]

airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml

@@ -3,11 +3,11 @@ data:
     ql: 200
     sl: 300
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.2.2@sha256:57703de3b4c4204bd68a7b13c9300f8e03c0189bffddaffc796f1da25d2dbea0
+    baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
   connectorSubtype: file
   connectorType: source
   definitionId: 31e3242f-dee7-4cdc-a4b8-8e06c5458517
-  dockerImageTag: 1.0.1
+  dockerImageTag: 1.1.0
   dockerRepository: airbyte/source-sftp-bulk
   documentationUrl: https://docs.airbyte.com/integrations/sources/sftp-bulk
   githubIssueLabel: source-sftp-bulk