🐛Source Rss: Fix Incremental Sync (#37535)

Author: btkcodedev

airbyte-integrations/connectors/source-rss/README.md

@@ -25,7 +25,7 @@ poetry install --with dev
 **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/rss)
 to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_rss/spec.yaml` file.
 Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
-See `sample_files/sample_config.json` for a sample config file.
+See `integration_tests/sample_config.json` for a sample config file.
 
 
 ### Locally running the connector
@@ -34,7 +34,7 @@ See `sample_files/sample_config.json` for a sample config file.
 poetry run source-rss spec
 poetry run source-rss check --config secrets/config.json
 poetry run source-rss discover --config secrets/config.json
-poetry run source-rss read --config secrets/config.json --catalog sample_files/configured_catalog.json
+poetry run source-rss read --config secrets/config.json --catalog integration_tests/configured_catalog.json
 ```
 
 ### Running tests
@@ -100,4 +100,4 @@ You've checked out the repo, implemented a million dollar feature, and you're re
 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
 6. Pat yourself on the back for being an awesome contributor.
 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
-8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry.
+8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry.

airbyte-integrations/connectors/source-rss/acceptance-test-config.yml

@@ -20,12 +20,11 @@ acceptance_tests:
         configured_catalog_path: "integration_tests/configured_catalog.json"
         empty_streams: []
   incremental:
-    bypass_reason: "This connector does not implement incremental sync"
-    # tests:
-    #   - config_path: "secrets/config.json"
-    #     configured_catalog_path: "integration_tests/configured_catalog.json"
-    #     future_state:
-    #       future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-rss/integration_tests/abnormal_state.json

@@ -1,5 +1,5 @@
 {
-  "todo-stream-name": {
-    "todo-field-name": "value"
+  "items": {
+    "published": "3333-10-24T16:16:00+00:00"
   }
 }

airbyte-integrations/connectors/source-rss/integration_tests/configured_catalog.json

@@ -3,8 +3,42 @@
     {
       "stream": {
         "name": "items",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "required": ["published"],
+          "properties": {
+            "title": {
+              "type": ["null", "string"]
+            },
+            "link": {
+              "type": ["null", "string"]
+            },
+            "description": {
+              "type": ["null", "string"]
+            },
+            "author": {
+              "type": ["null", "string"]
+            },
+            "category": {
+              "type": ["null", "string"]
+            },
+            "comments": {
+              "type": ["null", "string"]
+            },
+            "enclosure": {
+              "type": ["null", "string"]
+            },
+            "guid": {
+              "type": ["null", "string"]
+            },
+            "published": {
+              "type": ["string"],
+              "format": "date-time"
+            }
+          }
+        },
+        "supported_sync_modes": ["full_refresh", "incremental"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"

airbyte-integrations/connectors/source-rss/integration_tests/sample_state.json

@@ -1,5 +1,5 @@
 {
-  "todo-stream-name": {
-    "todo-field-name": "value"
+  "items": {
+    "published": "2022-10-24T16:16:00+00:00"
   }
 }

airbyte-integrations/connectors/source-rss/metadata.yaml

@@ -24,7 +24,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 0efee448-6948-49e2-b786-17db50647908
-  dockerImageTag: 1.0.0
+  dockerImageTag: 1.0.1
   dockerRepository: airbyte/source-rss
   githubIssueLabel: source-rss
   icon: rss.svg

airbyte-integrations/connectors/source-rss/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.0.0"
+version = "1.0.1"
 name = "source-rss"
 description = "Source implementation for rss."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-rss/source_rss/manifest.yaml

@@ -1,4 +1,4 @@
-version: "0.44.0"
+version: "0.78.5"
 
 definitions:
   selector:
@@ -17,6 +17,8 @@ definitions:
     type: SimpleRetriever
     record_selector:
       $ref: "#/definitions/selector"
+      record_filter:
+        condition: "{{ record['published'] >= stream_interval['start_time'] }}"
     paginator:
       type: NoPagination
     requester:
@@ -36,6 +38,18 @@ definitions:
         $ref: "#/definitions/items_schema"
     $parameters:
       path: "/"
+    incremental_sync:
+      type: DatetimeBasedCursor
+      cursor_field: published
+      datetime_format: "%Y-%m-%dT%H:%M:%S%z"
+      start_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ (now_utc() - duration('PT23H')).strftime('%Y-%m-%dT%H:%M:%S%z') }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%S%z"
+      end_datetime:
+        type: MinMaxDatetime
+        datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%S%z') }}"
+        datetime_format: "%Y-%m-%dT%H:%M:%S%z"
 
   items_schema:
     $schema: http://json-schema.org/draft-07/schema#

docs/integrations/sources/rss-migrations.md

@@ -1,4 +1,4 @@
-# RSS Migration Guide
+# Rss Migration Guide
 
 ## Upgrading to 1.0.0
 We're continuously striving to enhance the quality and reliability of our connectors at Airbyte.
@@ -18,4 +18,4 @@ Clearing your data is required for the affected streams in order to continue syn
     1. Ensure the **Clear affected streams** option is checked to ensure your streams continue syncing successfully with the new schema.
 4. Select **Save connection**. 
 
-This will clear the data in your destination for the subset of streams with schema changes. After the clear succeeds, trigger a sync by clicking **Sync Now**. For more information on clearing your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset).
+This will clear the data in your destination for the subset of streams with schema changes. After the clear succeeds, trigger a sync by clicking **Sync Now**. For more information on clearing your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset).

docs/integrations/sources/rss.md

@@ -34,5 +34,6 @@ None
 
 | Version | Date        | Pull Request                                             | Subject                        |
 | :------ | :---------- | :------------------------------------------------------- | :----------------------------- |
+| 1.0.1   | 2024-04-30  | [37535](https://github.com/airbytehq/airbyte/pull/37535) | Fix incremental sync           |
 | 1.0.0   | 2024-04-20  | [36418](https://github.com/airbytehq/airbyte/pull/36418) | Migrate python cdk to low code |
 | 0.1.0   | 2022-10-12  | [18838](https://github.com/airbytehq/airbyte/pull/18838) | Initial release supporting RSS |