File-based CDK: avoid error on empty stream when running discover (#38230)

Author: Anton Karpets

airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -16,7 +16,6 @@
     FileBasedSourceError,
     InvalidSchemaError,
     MissingSchemaError,
-    NoFilesMatchingError,
     RecordParseError,
     SchemaInferenceError,
     StopSyncPerValidationPolicy,
@@ -172,7 +171,7 @@ def get_json_schema(self) -> JsonSchema:
         }
         try:
             schema = self._get_raw_json_schema()
-        except (InvalidSchemaError, NoFilesMatchingError) as config_exception:
+        except InvalidSchemaError as config_exception:
             self.logger.exception(FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, exc_info=config_exception)
             raise AirbyteTracedException(
                 internal_message="Please check the logged errors for more information.",
@@ -195,7 +194,8 @@ def _get_raw_json_schema(self) -> JsonSchema:
             total_n_files = len(files)
 
             if total_n_files == 0:
-                raise NoFilesMatchingError(FileBasedSourceError.EMPTY_STREAM, stream=self.name)
+                self.logger.warning(msg=f"No files were identified in the stream {self.name}. Setting default schema for the stream.")
+                return schemaless_schema
 
             max_n_files_for_schema_inference = self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser())
             if total_n_files > max_n_files_for_schema_inference:

airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -3029,7 +3029,6 @@
         .set_file_type("csv")
     )
     .set_expected_check_status("FAILED")
-    .set_expected_check_error(AirbyteTracedException, FileBasedSourceError.EMPTY_STREAM.value)
     .set_expected_catalog(
         {
             "streams": [
@@ -3038,10 +3037,9 @@
                     "json_schema": {
                         "type": "object",
                         "properties": {
-                            "col1": {"type": "string"},
-                            "col2": {"type": "string"},
                             "_ab_source_file_last_modified": {"type": "string"},
                             "_ab_source_file_url": {"type": "string"},
+                            "data": {"type": "object"},
                         },
                     },
                     "name": "stream1",
@@ -3051,7 +3049,7 @@
             ]
         }
     )
-    .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
+    .set_expected_records(None)
 ).build()
 
 csv_no_records_scenario: TestScenario[InMemoryFilesSource] = (
@@ -3109,3 +3107,48 @@
     )
     .set_expected_records([])
 ).build()
+
+csv_no_files_scenario: TestScenario[InMemoryFilesSource] = (
+    TestScenarioBuilder[InMemoryFilesSource]()
+    .set_name("no_files_csv_stream")
+    .set_config(
+        {
+            "streams": [
+                {
+                    "name": "stream1",
+                    "format": {"filetype": "csv"},
+                    "globs": ["*"],
+                    "validation_policy": "Emit Record",
+                }
+            ],
+            "start_date": "2023-06-10T03:54:07.000000Z",
+        }
+    )
+    .set_source_builder(
+        FileBasedSourceBuilder()
+        .set_files({})
+        .set_file_type("csv")
+    )
+    .set_expected_check_status("FAILED")
+    .set_expected_catalog(
+        {
+            "streams": [
+                {
+                    "default_cursor_field": ["_ab_source_file_last_modified"],
+                    "json_schema": {
+                        "type": "object",
+                        "properties": {
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                            "data": {"type": "object"},
+                        },
+                    },
+                    "name": "stream1",
+                    "source_defined_cursor": True,
+                    "supported_sync_modes": ["full_refresh", "incremental"],
+                }
+            ]
+        }
+    )
+    .set_expected_records(None)
+).build()

airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py

@@ -62,6 +62,7 @@
     csv_multi_stream_scenario,
     csv_newline_in_values_not_quoted_scenario,
     csv_newline_in_values_quoted_value_scenario,
+    csv_no_files_scenario,
     csv_no_records_scenario,
     csv_single_stream_scenario,
     csv_skip_after_header_scenario,
@@ -154,7 +155,6 @@
 from unit_tests.sources.file_based.test_scenarios import verify_check, verify_discover, verify_read, verify_spec
 
 discover_failure_scenarios = [
-    earlier_csv_scenario,
     empty_schema_inference_scenario,
 ]
 
@@ -263,6 +263,8 @@
     single_csv_input_state_is_earlier_scenario_concurrent,
     single_csv_input_state_is_later_scenario_concurrent,
     single_csv_no_input_state_scenario_concurrent,
+    earlier_csv_scenario,
+    csv_no_files_scenario,
 ]
 
 discover_scenarios = discover_failure_scenarios + discover_success_scenarios
@@ -296,6 +298,7 @@
     valid_single_stream_user_input_schema_scenario,
     single_avro_scenario,
     earlier_csv_scenario,
+    csv_no_files_scenario,
 ]