From c8b42e0a0b164682797c50eba678189c6efc145e Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 7 Mar 2022 17:17:51 +0100 Subject: [PATCH 01/34] Added source for RKI-covid-germany, updated spec.json, implemented source with check and discover method added germany.json. --- .../connectors/source-rki-covid/.dockerignore | 6 + .../connectors/source-rki-covid/Dockerfile | 38 ++++ .../connectors/source-rki-covid/README.md | 132 +++++++++++ .../acceptance-test-config.yml | 30 +++ .../acceptance-test-docker.sh | 16 ++ .../connectors/source-rki-covid/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 ++++ .../integration_tests/configured_catalog.json | 22 ++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-rki-covid/main.py | 13 ++ .../source-rki-covid/requirements.txt | 2 + .../connectors/source-rki-covid/setup.py | 29 +++ .../source_rki_covid/__init__.py | 8 + .../source_rki_covid/schemas/TODO.md | 25 +++ .../source_rki_covid/schemas/germany.json | 1 + .../source_rki_covid/source.py | 209 ++++++++++++++++++ .../source_rki_covid/spec.json | 16 ++ .../source-rki-covid/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++ .../unit_tests/test_source.py | 22 ++ .../unit_tests/test_streams.py | 83 +++++++ 26 files changed, 797 insertions(+) create mode 100644 airbyte-integrations/connectors/source-rki-covid/.dockerignore create mode 100644 airbyte-integrations/connectors/source-rki-covid/Dockerfile create mode 100644 airbyte-integrations/connectors/source-rki-covid/README.md create mode 100644 airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-rki-covid/build.gradle create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/main.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/requirements.txt create mode 100644 airbyte-integrations/connectors/source-rki-covid/setup.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/__init__.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-rki-covid/.dockerignore b/airbyte-integrations/connectors/source-rki-covid/.dockerignore new file mode 100644 index 0000000000000..0e23ce64d7c7a --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_rki_covid +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile new file mode 100644 index 0000000000000..3376be9a98ad5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_rki_covid ./source_rki_covid + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-rki-covid diff --git a/airbyte-integrations/connectors/source-rki-covid/README.md b/airbyte-integrations/connectors/source-rki-covid/README.md new file mode 100644 index 0000000000000..fb26dcc3b4eeb --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/README.md @@ -0,0 +1,132 @@ +# Rki Covid Source + +This is the repository for the Rki Covid source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rki-covid). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-rki-covid:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rki-covid) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rki_covid/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source rki-covid test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-rki-covid:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-rki-covid:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-rki-covid:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rki-covid:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rki-covid:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rki-covid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-rki-covid:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-rki-covid:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml new file mode 100644 index 0000000000000..57289e2f271a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-rki-covid:dev +tests: + spec: + - spec_path: "source_rki_covid/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-rki-covid/build.gradle b/airbyte-integrations/connectors/source-rki-covid/build.gradle new file mode 100644 index 0000000000000..eb34765a96db5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_rki_covid' +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/__init__.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py new file mode 100644 index 0000000000000..056971f954502 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json new file mode 100644 index 0000000000000..6799946a68514 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..36f0468db0d8f --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f3732995784f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json new file mode 100644 index 0000000000000..ecc4913b84c74 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-rki-covid/main.py b/airbyte-integrations/connectors/source-rki-covid/main.py new file mode 100644 index 0000000000000..599f0ebf6562a --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_rki_covid import SourceRkiCovid + +if __name__ == "__main__": + source = SourceRkiCovid() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-rki-covid/requirements.txt b/airbyte-integrations/connectors/source-rki-covid/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py new file mode 100644 index 0000000000000..0d45f842d3e0a --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_rki_covid", + description="Source implementation for Rki Covid.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/__init__.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/__init__.py new file mode 100644 index 0000000000000..13bc66fa24171 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceRkiCovid + +__all__ = ["SourceRkiCovid"] diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/TODO.md b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/TODO.md new file mode 100644 index 0000000000000..cf1efadb3c9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json new file mode 100644 index 0000000000000..cd838b986cb62 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py new file mode 100644 index 0000000000000..49ac02fc37003 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream +class RkiCovidStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class RkiCovidStream(HttpStream, ABC)` which is the current class + `class Customers(RkiCovidStream)` contains behavior to pull data for customers using v1/customers + `class Employees(RkiCovidStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalRkiCovidStream((RkiCovidStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://api.corona-zahlen.org/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Germany(RkiCovidStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class IncrementalRkiCovidStream(RkiCovidStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalRkiCovidStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourceRkiCovid(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + req = requests.get(RkiCovidStream.url_base+'germany') + if req.status_code == 200: + return True, None + return False, req.reason + + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + return [Germany()] diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json new file mode 100644 index 0000000000000..f40175a6b1264 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -0,0 +1,16 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Rki Covid Spec", + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "history_in_days": { + "type": "integer", + "description": "Number of days in the past from today." + } + } + } +} diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/__init__.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..5698d931fa731 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_rki_covid.source import IncrementalRkiCovidStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalRkiCovidStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalRkiCovidStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalRkiCovidStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalRkiCovidStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalRkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalRkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalRkiCovidStream, "cursor_field", "dummy_field") + stream = IncrementalRkiCovidStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalRkiCovidStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalRkiCovidStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py new file mode 100644 index 0000000000000..b5da106d582fb --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_rki_covid.source import SourceRkiCovid + + +def test_check_connection(mocker): + source = SourceRkiCovid() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceRkiCovid() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py new file mode 100644 index 0000000000000..68ac98d9e57ec --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_rki_covid.source import RkiCovidStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(RkiCovidStream, "path", "v0/example_endpoint") + mocker.patch.object(RkiCovidStream, "primary_key", "test_primary_key") + mocker.patch.object(RkiCovidStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = RkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = RkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = RkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = RkiCovidStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = RkiCovidStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = RkiCovidStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = RkiCovidStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From 2c93e79c24853af30d91336def9b9e9bbbcc89a6 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Thu, 10 Mar 2022 18:02:09 +0100 Subject: [PATCH 02/34] implemented incremental method for germany history cases with date as parameters, updated streams, added cursor field for incremental streams. --- .../schemas/germany_history_cases.json | 1 + .../source_rki_covid/source.py | 52 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json new file mode 100644 index 0000000000000..fb9c6ecfd7d9b --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"cases": {"type": "integer"}, "date": {"type": "string"}}, "required": ["cases", "date"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 49ac02fc37003..7b7c76b69accc 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -89,7 +89,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ - yield {} + yield response.json() class Germany(RkiCovidStream): @@ -98,7 +98,7 @@ class Germany(RkiCovidStream): """ # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" + primary_key = None def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -107,7 +107,7 @@ def path( TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this should return "customers". Required. """ - return "customers" + return "germany" # Basic incremental stream @@ -138,6 +138,47 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return {} +class GermanyHistoryCases(IncrementalRkiCovidStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('history_in_days'): + return "germany/history/cases/"+str(self.config.get('history_in_days')) + return "germany/history/cases/" class Employees(IncrementalRkiCovidStream): """ @@ -145,7 +186,7 @@ class Employees(IncrementalRkiCovidStream): """ # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" + cursor_field = "date" # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = "employee_id" @@ -206,4 +247,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ # TODO remove the authenticator if not required. - return [Germany()] + + return [Germany(), GermanyHistoryCases(config=config)] From 55e513f1cc5f9135ae40cc0d8c582ee10bd21714 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Thu, 10 Mar 2022 18:04:31 +0100 Subject: [PATCH 03/34] main file added. --- main.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 0000000000000..94e3a87232a22 --- /dev/null +++ b/main.py @@ -0,0 +1,16 @@ +# This is a sample Python script. + +# Press ⌃R to execute it or replace it with your code. +# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings. + + +def print_hi(name): + # Use a breakpoint in the code line below to debug your script. + print(f'Hi, {name}') # Press ⌘F8 to toggle the breakpoint. + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + print_hi('PyCharm') + +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ From 5041be2d78d6d75a9a0a7ed00aed407dd457b493 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Sun, 13 Mar 2022 13:12:38 +0100 Subject: [PATCH 04/34] added Incidence source with date as parameter. Incremental stream. spec, source, schemas updated, added class GermanHistoryIncidence added. --- .../schemas/german_history_incidence.json | 1 + .../source_rki_covid/source.py | 44 ++++++++++++++++++- .../source_rki_covid/spec.json | 4 ++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json new file mode 100644 index 0000000000000..ccdbf229b1626 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 7b7c76b69accc..d7298a7cdf09f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -180,6 +180,46 @@ def path( return "germany/history/cases/"+str(self.config.get('history_in_days')) return "germany/history/cases/" +class GermanHistoryIncidence(IncrementalRkiCovidStream): + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('incidence_in_days'): + return "germany/history/incidence/"+str(self.config.get('incidence_in_days')) + return "germany/history/incidence/" + + class Employees(IncrementalRkiCovidStream): """ TODO: Change class name to match the table/data source this stream corresponds to. @@ -248,4 +288,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ # TODO remove the authenticator if not required. - return [Germany(), GermanyHistoryCases(config=config)] + return [Germany(), + GermanyHistoryCases(config=config), + GermanHistoryIncidence(config=config)] diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json index f40175a6b1264..0d717e5741952 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -10,6 +10,10 @@ "history_in_days": { "type": "integer", "description": "Number of days in the past from today." + }, + "incidence_in_days": { + "type": "integer", + "description": "Number of days in the past from today." } } } From ae218200ec96a2d039bc168522d937850ec1cf59 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 14 Mar 2022 16:52:10 +0100 Subject: [PATCH 05/34] Added a full-refresh stream for germany/age-group .Added incremental streams for deaths/:days, recovered/:days, frozen-incidence/:days, hospitalization/:days. Updated source.py methods. Updated sepc.json properties key. Updated configured_catalogue.json with required streams. Updated config.json --- .../schemas/german_history_deaths.json | 1 + .../german_history_frozen_incidence.json | 1 + .../german_history_hospitalization.json | 1 + .../schemas/german_history_recovered.json | 1 + .../schemas/germany_age_groups.json | 1 + .../source_rki_covid/source.py | 203 +++++++++++++++++- .../source_rki_covid/spec.json | 22 +- 7 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json create mode 100644 airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json new file mode 100644 index 0000000000000..4e32e429e3f31 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"deaths": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "deaths"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json new file mode 100644 index 0000000000000..12abaadb0a228 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "object", "properties": {"abbreviation": {"type": "string"}, "name": {"type": "string"}, "history": {"type": "array", "items": {"type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]}}}, "required": ["abbreviation", "history", "name"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json new file mode 100644 index 0000000000000..0c11ab30f2c02 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "fixedCases7Days": {"type": ["integer", "null"]}, "updatedCases7Days": {"type": ["integer", "null"]}, "adjustedLowerCases7Days": {"type": ["integer", "null"]}, "adjustedCases7Days": {"type": ["integer", "null"]}, "adjustedUpperCases7Days": {"type": ["integer", "null"]}, "fixedIncidence7Days": {"type": ["null", "number"]}, "updatedIncidence7Days": {"type": ["null", "number"]}, "adjustedLowerIncidence7Days": {"type": ["null", "number"]}, "adjustedIncidence7Days": {"type": ["null", "number"]}, "adjustedUpperIncidence7Days": {"type": ["null", "number"]}}, "required": ["adjustedCases7Days", "adjustedIncidence7Days", "adjustedLowerCases7Days", "adjustedLowerIncidence7Days", "adjustedUpperCases7Days", "adjustedUpperIncidence7Days", "cases7Days", "date", "fixedCases7Days", "fixedIncidence7Days", "incidence7Days", "updatedCases7Days", "updatedIncidence7Days"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json new file mode 100644 index 0000000000000..32b957fe7746e --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"recovered": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "recovered"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json new file mode 100644 index 0000000000000..43b8b39a82dea --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -0,0 +1 @@ +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "object", "properties": {"A00-A04": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A05-A14": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A15-A34": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A35-A59": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "integer"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A60-A79": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A80+": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "integer"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}}, "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index d7298a7cdf09f..3144893620073 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -92,6 +92,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield response.json() +# class that contains main source germany | full-refresh class Germany(RkiCovidStream): """ TODO: Change class name to match the table/data source this stream corresponds to. @@ -110,6 +111,25 @@ def path( return "germany" +# class that contains source age-groups in germany. | full-refresh +class GermanyAgeGroups(RkiCovidStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = None + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "germany/age-groups" + + # Basic incremental stream class IncrementalRkiCovidStream(RkiCovidStream, ABC): """ @@ -138,6 +158,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return {} + +# source: germany/history/cases/:days | Incremental class GermanyHistoryCases(IncrementalRkiCovidStream): """ TODO: Change class name to match the table/data source this stream corresponds to. @@ -176,10 +198,13 @@ def path( TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this should return "customers". Required. """ - if self.config.get('history_in_days'): - return "germany/history/cases/"+str(self.config.get('history_in_days')) + if self.config.get('cases_in_days'): + return "germany/history/cases/"+str(self.config.get('cases_in_days')) return "germany/history/cases/" + + +# source: germany/history/incidence/:days | Incremental class GermanHistoryIncidence(IncrementalRkiCovidStream): primary_key = None @@ -220,6 +245,173 @@ def path( return "germany/history/incidence/" + +# source: germany/history/deaths/:days | Incremental +class GermanHistoryDeaths(IncrementalRkiCovidStream): + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('deaths_in_days'): + return "germany/history/deaths/"+str(self.config.get('deaths_in_days')) + return "germany/history/deaths/" + + + +# source: germany/history/recovered/:days | Incremental +class GermanHistoryRecovered(IncrementalRkiCovidStream): + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('recovered_in_days'): + return "germany/history/recovered/"+str(self.config.get('recovered_in_days')) + return "germany/history/recovered/" + + + +# source: germany/history/frozen-incidence/:days | Incremental +class GermanHistoryFrozenIncidence(IncrementalRkiCovidStream): + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data").get("history") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('frozen_incidence_in_days'): + return "germany/history/frozen-incidence/"+str(self.config.get('frozen_incidence_in_days')) + return "germany/history/frozen-incidence/" + + +# source: germany/history/hospitalization/:days | Incremental +class GermanHistoryHospitalization(IncrementalRkiCovidStream): + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return "date" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json().get("data") + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + if self.config.get('hospitalization_in_days'): + return "germany/history/hospitalization/"+str(self.config.get('hospitalization_in_days')) + return "germany/history/hospitalization/" + + class Employees(IncrementalRkiCovidStream): """ TODO: Change class name to match the table/data source this stream corresponds to. @@ -289,5 +481,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # TODO remove the authenticator if not required. return [Germany(), + GermanyAgeGroups(), GermanyHistoryCases(config=config), - GermanHistoryIncidence(config=config)] + GermanHistoryIncidence(config=config), + GermanHistoryDeaths(config=config), + GermanHistoryRecovered(config=config), + GermanHistoryFrozenIncidence(config=config), + GermanHistoryHospitalization(config=config)] diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json index 0d717e5741952..64544cd268ead 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -7,13 +7,29 @@ "required": [], "additionalProperties": false, "properties": { - "history_in_days": { + "cases_in_days": { "type": "integer", - "description": "Number of days in the past from today." + "description": "Number of cases days in the past from today." }, "incidence_in_days": { "type": "integer", - "description": "Number of days in the past from today." + "description": "Number of incidence days in the past from today." + }, + "deaths_in_days": { + "type": "integer", + "description": "Number of deaths days in the past from today." + }, + "recovered_in_days": { + "type": "integer", + "description": "Number of recovered days in the past from today." + }, + "frozen_incidence_in_days": { + "type": "integer", + "description": "Number of frozen-incidence days in the past from today." + }, + "hospitalization_in_days": { + "type": "integer", + "description": "Number of hospitalization days in the past from today." } } } From d40604a111f815b51555b262abd91495e12158f7 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 21 Mar 2022 17:52:51 +0100 Subject: [PATCH 06/34] writting test unit test cases for incremental streams: german history cases and german history incidence. --- .../test_incremental_germanhistorycases.py | 54 +++++++++++++++++++ ...test_incremental_germanhistoryincidence.py | 54 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py new file mode 100644 index 0000000000000..9c4f268ee7aa0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanyHistoryCases + +@fixture +def patch_incremental_german_history_cases(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanyHistoryCases, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_cases): + config = {"cases_in_days": 2} + stream = GermanyHistoryCases(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_cases): + config = {"cases_in_days": 2} + stream = GermanyHistoryCases(config) + d = datetime.date(datetime.today()) - timedelta(days=2) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_cases): + config = {"cases_in_days": 2} + stream = GermanyHistoryCases(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/cases/1') + expected_response = response.json().get("data") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_cases): + config = {"cases_in_days": 2} + stream = GermanyHistoryCases(config) + expected_stream_path = "germany/history/cases/"+str(config.get('cases_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_cases): + config = {} + stream = GermanyHistoryCases(config) + expected_stream_path = "germany/history/cases/" + assert stream.path() == expected_stream_path \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py new file mode 100644 index 0000000000000..b7aa3de897718 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryIncidence + +@fixture +def patch_incremental_german_history_incidence(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanHistoryIncidence, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_incidence): + config = {"incidence_in_days": 2} + stream = GermanHistoryIncidence(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_incidence): + config = {"incidence_in_days": 2} + stream = GermanHistoryIncidence(config) + d = datetime.date(datetime.today()) - timedelta(days=2) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_incidence): + config = {"incidence_in_days": 2} + stream = GermanHistoryIncidence(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/incidence/1') + expected_response = response.json().get("data") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_incidence): + config = {"incidence_in_days": 2} + stream = GermanHistoryIncidence(config) + expected_stream_path = "germany/history/incidence/"+str(config.get('incidence_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_incidence): + config = {} + stream = GermanHistoryIncidence(config) + expected_stream_path = "germany/history/incidence/" + assert stream.path() == expected_stream_path \ No newline at end of file From c383fb722fe7eae778f2030e387f1734875673f0 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 21 Mar 2022 22:29:29 +0100 Subject: [PATCH 07/34] Incremental streams for germanhistorydeaths, germanhistoryfrozenIncidence, germanhistoryhospitalization, germanhistoryrecovered. Fixing other test cases. --- .../test_incremental_germanhistorydeaths.py | 54 +++++++++++++++++++ ...ncremental_germanhistoryfrozenIncidence.py | 54 +++++++++++++++++++ ...ncremental_germanhistoryhospitalization.py | 54 +++++++++++++++++++ ...test_incremental_germanhistoryincidence.py | 2 +- ...test_incremental_germanhistoryrecovered.py | 54 +++++++++++++++++++ .../unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 14 ++--- 7 files changed, 225 insertions(+), 9 deletions(-) create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py new file mode 100644 index 0000000000000..9382e46f5b8d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryDeaths + +@fixture +def patch_incremental_german_history_deaths(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanHistoryDeaths, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_deaths): + config = {"deaths_in_days": 2} + stream = GermanHistoryDeaths(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_deaths): + config = {"deaths_in_days": 2} + stream = GermanHistoryDeaths(config) + d = datetime.date(datetime.today()) - timedelta(days=1) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_deaths): + config = {"deaths_in_days": 2} + stream = GermanHistoryDeaths(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/deaths/1') + expected_response = response.json().get("data") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_deaths): + config = {"deaths_in_days": 2} + stream = GermanHistoryDeaths(config) + expected_stream_path = "germany/history/deaths/"+str(config.get('deaths_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_deaths): + config = {} + stream = GermanHistoryDeaths(config) + expected_stream_path = "germany/history/deaths/" + assert stream.path() == expected_stream_path \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py new file mode 100644 index 0000000000000..4d45e23984b80 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryFrozenIncidence + +@fixture +def patch_incremental_german_history_frozenInc(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanHistoryFrozenIncidence, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_frozenInc): + config = {"frozen_incidence_in_days": 2} + stream = GermanHistoryFrozenIncidence(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_frozenInc): + config = {"frozen_incidence_in_days": 2} + stream = GermanHistoryFrozenIncidence(config) + d = datetime.date(datetime.today()) - timedelta(days=1) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_frozenInc): + config = {"frozen_incidence_in_days": 2} + stream = GermanHistoryFrozenIncidence(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/frozen-incidence/1') + expected_response = response.json().get("data").get("history") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_frozenInc): + config = {"frozen_incidence_in_days": 2} + stream = GermanHistoryFrozenIncidence(config) + expected_stream_path = "germany/history/frozen-incidence/"+str(config.get('frozen_incidence_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_frozenInc): + config = {} + stream = GermanHistoryFrozenIncidence(config) + expected_stream_path = "germany/history/frozen-incidence/" + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py new file mode 100644 index 0000000000000..3950cab7b442e --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryHospitalization + +@fixture +def patch_incremental_german_history_hospitalization(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanHistoryHospitalization, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_hospitalization): + config = {"hospitalization_in_days": 2} + stream = GermanHistoryHospitalization(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_hospitalization): + config = {"hospitalization_in_days": 2} + stream = GermanHistoryHospitalization(config) + d = datetime.date(datetime.today()) - timedelta(days=1) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_hospitalization): + config = {"hospitalization_in_days": 2} + stream = GermanHistoryHospitalization(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/hospitalization/1') + expected_response = response.json().get("data") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_hospitalization): + config = {"hospitalization_in_days": 2} + stream = GermanHistoryHospitalization(config) + expected_stream_path = "germany/history/hospitalization/"+str(config.get('hospitalization_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_hospitalization): + config = {} + stream = GermanHistoryHospitalization(config) + expected_stream_path = "germany/history/hospitalization/" + assert stream.path() == expected_stream_path \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py index b7aa3de897718..2f291a436fbe6 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py @@ -25,7 +25,7 @@ def test_cursor_field(patch_incremental_german_history_incidence): def test_get_updated_state(patch_incremental_german_history_incidence): config = {"incidence_in_days": 2} stream = GermanHistoryIncidence(config) - d = datetime.date(datetime.today()) - timedelta(days=2) + d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} inputs = {"current_stream_state": date, "latest_record": date} expected_state = {stream.cursor_field: str(d)} diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py new file mode 100644 index 0000000000000..674adc5ba81a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from datetime import datetime, timedelta +from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryRecovered + +@fixture +def patch_incremental_german_history_recovered(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanHistoryRecovered, "primary_key", None) + + +def test_cursor_field(patch_incremental_german_history_recovered): + config = {"recovered_in_days": 2} + stream = GermanHistoryRecovered(config) + expected_cursor_field = "date" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_german_history_recovered): + config = {"recovered_in_days": 2} + stream = GermanHistoryRecovered(config) + d = datetime.date(datetime.today()) - timedelta(days=1) + date = {stream.cursor_field: str(d)} + inputs = {"current_stream_state": date, "latest_record": date} + expected_state = {stream.cursor_field: str(d)} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_parse_response(patch_incremental_german_history_recovered): + config = {"recovered_in_days": 2} + stream = GermanHistoryRecovered(config) + response = requests.get('https://api.corona-zahlen.org/germany/history/recovered/1') + expected_response = response.json().get("data") + assert stream.parse_response(response) == expected_response + + +def test_parse_with_cases(patch_incremental_german_history_recovered): + config = {"recovered_in_days": 2} + stream = GermanHistoryRecovered(config) + expected_stream_path = "germany/history/recovered/"+str(config.get('recovered_in_days')) + assert stream.path() == expected_stream_path + + +def test_parse_without_cases(patch_incremental_german_history_recovered): + config = {} + stream = GermanHistoryRecovered(config) + expected_stream_path = "germany/history/recovered/" + assert stream.path() == expected_stream_path \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py index b5da106d582fb..30edf62f33107 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py @@ -18,5 +18,5 @@ def test_streams(mocker): config_mock = MagicMock() streams = source.streams(config_mock) # TODO: replace this with your streams number - expected_streams_number = 2 + expected_streams_number = 8 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py index 68ac98d9e57ec..e85155a3f862c 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py @@ -35,13 +35,13 @@ def test_next_page_token(patch_base_class): assert stream.next_page_token(**inputs) == expected_token -def test_parse_response(patch_base_class): - stream = RkiCovidStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object +# def test_parse_response(patch_base_class): +# stream = RkiCovidStream() +# # TODO: replace this with your input parameters +# inputs = {"response": MagicMock()} +# # TODO: replace this with your expected parced object +# expected_parsed_object = {} +# assert next(stream.parse_response(**inputs)) == expected_parsed_object def test_request_headers(patch_base_class): From b2f8fee865d30d8a8b349744cbcb24e9f1c08afc Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Wed, 23 Mar 2022 11:45:49 +0100 Subject: [PATCH 08/34] Added test stream for age group and germany. --- .../unit_tests/test_stream_agegroup.py | 21 +++++++++++++++++++ .../unit_tests/test_stream_germany.py | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py new file mode 100644 index 0000000000000..5e262c4d8f035 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_rki_covid.source import RkiCovidStream, GermanyAgeGroups + + +@pytest.fixture +def patch_age_group(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanyAgeGroups, "primary_key", None) + + +def test_path(patch_age_group): + stream = GermanyAgeGroups() + expected_params = {"path": "germany/age-groups"} + assert stream.path() == expected_params.get("path") \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py new file mode 100644 index 0000000000000..1695eec0c17e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_rki_covid.source import RkiCovidStream, Germany + + +@pytest.fixture +def patch_germany_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(Germany, "primary_key", None) + + +def test_path(patch_germany_class): + stream = Germany() + expected_params = {"path": "germany"} + assert stream.path() == expected_params.get("path") \ No newline at end of file From 51cc12180140a3871b96dcfc0bce91fadc6eb16e Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Fri, 25 Mar 2022 12:19:11 +0100 Subject: [PATCH 09/34] changes in Readme and source.py. --- .../connectors/source-rki-covid/README.md | 16 +- .../source_rki_covid/source.py | 172 ++++-------------- .../unit_tests/test_cached_stream_state.py | 0 3 files changed, 50 insertions(+), 138 deletions(-) create mode 100644 airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py diff --git a/airbyte-integrations/connectors/source-rki-covid/README.md b/airbyte-integrations/connectors/source-rki-covid/README.md index fb26dcc3b4eeb..833f6bc05c896 100644 --- a/airbyte-integrations/connectors/source-rki-covid/README.md +++ b/airbyte-integrations/connectors/source-rki-covid/README.md @@ -1,9 +1,21 @@ -# Rki Covid Source +# RKI Covid Source -This is the repository for the Rki Covid source connector, written in Python. +This is the repository for the RkI (Robert Koch-Institut - von Marlon Lückert) Covid-19 source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rki-covid). ## Local development +### Developed Streams (Endpoints) +``` +Germany: + 1. /germany + 2. /germany/age-groups + 3. /germany/history/cases/:days + 4. /germany/history/incidence/:days + 5. /germany/history/deaths/:days + 6. /germany/history/recovered/:days + 7. /germany/history/frozen-incidence/:days + 8. /germany/history/hospitalization/:days +``` ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 3144893620073..32cb7322b70c8 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -12,50 +12,11 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.json file. -""" - # Basic full refresh stream class RkiCovidStream(HttpStream, ABC): - """ - TODO remove this comment - - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class RkiCovidStream(HttpStream, ABC)` which is the current class - `class Customers(RkiCovidStream)` contains behavior to pull data for customers using v1/customers - `class Employees(RkiCovidStream)` contains behavior to pull data for employees using v1/employees - If some streams implement incremental sync, it is typical to create another class - `class IncrementalRkiCovidStream((RkiCovidStream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - See the reference docs for the full list of configurable options. - """ - - # TODO: Fill in the url base. Required. url_base = "https://api.corona-zahlen.org/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -94,50 +55,31 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp # class that contains main source germany | full-refresh class Germany(RkiCovidStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ + """Docs: https://api.corona-zahlen.org/germany""" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = None def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ return "germany" # class that contains source age-groups in germany. | full-refresh class GermanyAgeGroups(RkiCovidStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ + """Docs: https://api.corona-zahlen.org/germany/age-groups""" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = None def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ return "germany/age-groups" # Basic incremental stream class IncrementalRkiCovidStream(RkiCovidStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. state_checkpoint_interval = None @property @@ -161,11 +103,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late # source: germany/history/cases/:days | Incremental class GermanyHistoryCases(IncrementalRkiCovidStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ + """Docs: https://api.corona-zahlen.org/germany/germany/history/cases/:days""" - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. primary_key = None def __init__(self, config, **kwargs): @@ -175,10 +114,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -188,24 +124,21 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_state = current_stream_state.get(self.cursor_field) or latest_state return {self.cursor_field: max(latest_state, current_state)} + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('cases_in_days'): return "germany/history/cases/"+str(self.config.get('cases_in_days')) return "germany/history/cases/" - # source: germany/history/incidence/:days | Incremental class GermanHistoryIncidence(IncrementalRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/germany/history/incidence/:days""" primary_key = None @@ -216,10 +149,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -236,18 +166,14 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('incidence_in_days'): return "germany/history/incidence/"+str(self.config.get('incidence_in_days')) return "germany/history/incidence/" - # source: germany/history/deaths/:days | Incremental class GermanHistoryDeaths(IncrementalRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/germany/history/deaths/:days""" primary_key = None @@ -258,10 +184,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -278,18 +201,14 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('deaths_in_days'): return "germany/history/deaths/"+str(self.config.get('deaths_in_days')) return "germany/history/deaths/" - # source: germany/history/recovered/:days | Incremental class GermanHistoryRecovered(IncrementalRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/germany/history/recovered/:days""" primary_key = None @@ -300,10 +219,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -320,18 +236,14 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('recovered_in_days'): return "germany/history/recovered/"+str(self.config.get('recovered_in_days')) return "germany/history/recovered/" - # source: germany/history/frozen-incidence/:days | Incremental class GermanHistoryFrozenIncidence(IncrementalRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/germany/history/frozen-incidence/:days""" primary_key = None @@ -342,10 +254,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -362,10 +271,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('frozen_incidence_in_days'): return "germany/history/frozen-incidence/"+str(self.config.get('frozen_incidence_in_days')) return "germany/history/frozen-incidence/" @@ -373,6 +278,7 @@ def path( # source: germany/history/hospitalization/:days | Incremental class GermanHistoryHospitalization(IncrementalRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/germany/history/hospitalization/:days""" primary_key = None @@ -383,10 +289,7 @@ def __init__(self, config, **kwargs): @property def cursor_field(self) -> str: """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - + date is cursor field in the stream. :return str: The name of the cursor field. """ return "date" @@ -403,10 +306,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ if self.config.get('hospitalization_in_days'): return "germany/history/hospitalization/"+str(self.config.get('hospitalization_in_days')) return "germany/history/hospitalization/" @@ -455,36 +354,37 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite # Source class SourceRkiCovid(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. + Testing connection availability for the connector. :param config: the user-input config object conforming to the connector's spec.json :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - req = requests.get(RkiCovidStream.url_base+'germany') - if req.status_code == 200: - return True, None - return False, req.reason + try: + req = requests.get(RkiCovidStream.url_base+'germany') + if req.status_code == 200: + return True, None + return False, req.reason + except Exception: + return False, "There is a problem in source check connection." def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ - TODO: Replace the streams below with your own streams. - + Defining streams to run. :param config: A Mapping of the user input configuration as defined in the connector spec. """ - # TODO remove the authenticator if not required. - - return [Germany(), - GermanyAgeGroups(), - GermanyHistoryCases(config=config), - GermanHistoryIncidence(config=config), - GermanHistoryDeaths(config=config), - GermanHistoryRecovered(config=config), - GermanHistoryFrozenIncidence(config=config), - GermanHistoryHospitalization(config=config)] + + return [ + Germany(), + GermanyAgeGroups(), + GermanyHistoryCases(config=config), + GermanHistoryIncidence(config=config), + GermanHistoryDeaths(config=config), + GermanHistoryRecovered(config=config), + GermanHistoryFrozenIncidence(config=config), + GermanHistoryHospitalization(config=config) + ] diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py new file mode 100644 index 0000000000000..e69de29bb2d1d From d8298067d487ff26d32fb2817dffa4aa5b18cbff Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 28 Mar 2022 21:08:21 +0200 Subject: [PATCH 10/34] IncrementalMixin added to class GermanyHistoryCases. AFter review IncrementalMixin will be implemented to all incremental classes. --- .../source_rki_covid/source.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 32cb7322b70c8..b6dbe20b749f6 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests +from datetime import datetime from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -62,7 +63,7 @@ class Germany(RkiCovidStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - return "germany" + return "germany/" # class that contains source age-groups in germany. | full-refresh @@ -110,6 +111,7 @@ class GermanyHistoryCases(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + self._cursor_value = None @property def cursor_field(self) -> str: @@ -119,11 +121,24 @@ def cursor_field(self) -> str: """ return "date" - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} - + @property + def state(self) -> Mapping[str, Any]: + return {self.cursor_field: str(self._cursor_value)} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = value[self.cursor_field] + + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + for record in super().read_records(*args, **kwargs): + current_stream_state = record.get(self.cursor_field) + # print("self._cursor_value:", self._cursor_value, "current_stream_state:", current_stream_state) + if self._cursor_value: + latest_state = record.get(self.cursor_field) + self._cursor_value = max(self._cursor_value, latest_state) + yield record + self._cursor_value = current_stream_state + assert self._cursor_value == current_stream_state def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -161,7 +176,9 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_state, current_state)} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json().get("data") + if response.json().get("data"): + return response.json().get("data") + pass def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -382,7 +399,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Germany(), GermanyAgeGroups(), GermanyHistoryCases(config=config), - GermanHistoryIncidence(config=config), + # GermanHistoryIncidence(config=config), GermanHistoryDeaths(config=config), GermanHistoryRecovered(config=config), GermanHistoryFrozenIncidence(config=config), From 7836aa0f0a18a2023262a5db6f2e9168e63c3fe5 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 4 Apr 2022 15:00:28 +0200 Subject: [PATCH 11/34] Implemented Incremental mixin or GermanHistory Cases connector. --- .../source_rki_covid/source.py | 85 ++++++------------- .../test_incremental_germanhistorycases.py | 10 --- 2 files changed, 28 insertions(+), 67 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index b6dbe20b749f6..a8ca09f9682bf 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -21,20 +21,6 @@ class RkiCovidStream(HttpStream, ABC): url_base = "https://api.corona-zahlen.org/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ return None def request_params( @@ -111,8 +97,13 @@ class GermanyHistoryCases(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + self.start_date = None self._cursor_value = None + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -132,7 +123,6 @@ def state(self, value: Mapping[str, Any]): def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: for record in super().read_records(*args, **kwargs): current_stream_state = record.get(self.cursor_field) - # print("self._cursor_value:", self._cursor_value, "current_stream_state:", current_stream_state) if self._cursor_value: latest_state = record.get(self.cursor_field) self._cursor_value = max(self._cursor_value, latest_state) @@ -140,6 +130,7 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: self._cursor_value = current_stream_state assert self._cursor_value == current_stream_state + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -161,6 +152,10 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -198,6 +193,10 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -233,6 +232,10 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -268,6 +271,10 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -303,6 +310,10 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config + @property + def source_defined_cursor(self) -> bool: + return False + @property def cursor_field(self) -> str: """ @@ -318,6 +329,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_state, current_state)} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + self.logger.info(f"Cursor Field is = {self.cursor_field}") return response.json().get("data") def path( @@ -328,47 +340,6 @@ def path( return "germany/history/hospitalization/" -class Employees(IncrementalRkiCovidStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the cursor_field. Required. - cursor_field = "date" - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" - - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. - """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. - - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. - - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") - - # Source class SourceRkiCovid(AbstractSource): @@ -399,7 +370,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Germany(), GermanyAgeGroups(), GermanyHistoryCases(config=config), - # GermanHistoryIncidence(config=config), + GermanHistoryIncidence(config=config), GermanHistoryDeaths(config=config), GermanHistoryRecovered(config=config), GermanHistoryFrozenIncidence(config=config), diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py index 9c4f268ee7aa0..ced7f2993301f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py @@ -22,16 +22,6 @@ def test_cursor_field(patch_incremental_german_history_cases): assert stream.cursor_field == expected_cursor_field -def test_get_updated_state(patch_incremental_german_history_cases): - config = {"cases_in_days": 2} - stream = GermanyHistoryCases(config) - d = datetime.date(datetime.today()) - timedelta(days=2) - date = {stream.cursor_field: str(d)} - inputs = {"current_stream_state": date, "latest_record": date} - expected_state = {stream.cursor_field: str(d)} - assert stream.get_updated_state(**inputs) == expected_state - - def test_parse_response(patch_incremental_german_history_cases): config = {"cases_in_days": 2} stream = GermanyHistoryCases(config) From 6299be9fcbf04265bb1509c37882764c53adfef3 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 4 Apr 2022 16:47:52 +0200 Subject: [PATCH 12/34] corrected changes. --- .../connectors/source-rki-covid/source_rki_covid/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index a8ca09f9682bf..52dcec12a7d57 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -97,7 +97,6 @@ class GermanyHistoryCases(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config - self.start_date = None self._cursor_value = None @property From 33f710b367d665c65e52650b67821ae87b6d06a1 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 4 Apr 2022 21:16:50 +0200 Subject: [PATCH 13/34] adding integration test --- .../integration_tests/acceptance.py | 3 +- .../integration_tests/configured_catalog.json | 68 +++++++++++++++++-- .../unit_tests/test_incremental_streams.py | 7 -- .../unit_tests/test_source.py | 1 - .../unit_tests/test_streams.py | 7 -- 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py index 056971f954502..c1f0803ff9a6e 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py @@ -11,6 +11,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies + diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index 36f0468db0d8f..480f747715e45 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -2,21 +2,75 @@ "streams": [ { "stream": { - "name": "customers", - "json_schema": {}, + "name": "germany", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append_dedup" }, { "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "name": "germany_age_groups", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "germany_history_cases", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_incidence", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_deaths", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_recovered", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_frozen_incidence", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_hospitalization", + "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, + "supported_sync_modes": ["incremental"] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append_dedup" } ] } diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py index 5698d931fa731..c712f1e8fc31a 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py @@ -10,7 +10,6 @@ @fixture def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class mocker.patch.object(IncrementalRkiCovidStream, "path", "v0/example_endpoint") mocker.patch.object(IncrementalRkiCovidStream, "primary_key", "test_primary_key") mocker.patch.object(IncrementalRkiCovidStream, "__abstractmethods__", set()) @@ -18,25 +17,20 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): stream = IncrementalRkiCovidStream() - # TODO: replace this with your expected cursor field expected_cursor_field = [] assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_base_class): stream = IncrementalRkiCovidStream() - # TODO: replace this with your input parameters inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state expected_state = {} assert stream.get_updated_state(**inputs) == expected_state def test_stream_slices(patch_incremental_base_class): stream = IncrementalRkiCovidStream() - # TODO: replace this with your input parameters inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list expected_stream_slice = [None] assert stream.stream_slices(**inputs) == expected_stream_slice @@ -54,6 +48,5 @@ def test_source_defined_cursor(patch_incremental_base_class): def test_stream_checkpoint_interval(patch_incremental_base_class): stream = IncrementalRkiCovidStream() - # TODO: replace this with your expected checkpoint interval expected_checkpoint_interval = None assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py index 30edf62f33107..e893c21397105 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py @@ -17,6 +17,5 @@ def test_streams(mocker): source = SourceRkiCovid() config_mock = MagicMock() streams = source.streams(config_mock) - # TODO: replace this with your streams number expected_streams_number = 8 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py index e85155a3f862c..086781b57a1fa 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py @@ -19,18 +19,14 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): stream = RkiCovidStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters expected_params = {} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): stream = RkiCovidStream() - # TODO: replace this with your input parameters inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token expected_token = None assert stream.next_page_token(**inputs) == expected_token @@ -46,16 +42,13 @@ def test_next_page_token(patch_base_class): def test_request_headers(patch_base_class): stream = RkiCovidStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers expected_headers = {} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): stream = RkiCovidStream() - # TODO: replace this with your expected http request method expected_method = "GET" assert stream.http_method == expected_method From 62e7df960e82f686cb0e91314047af82496c4770 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Tue, 5 Apr 2022 18:25:33 +0200 Subject: [PATCH 14/34] comment acceptence test. --- .../acceptance-test-config.yml | 28 ++++++------- .../acceptance-test-docker.sh | 0 .../connectors/source-rki-covid/bootstrap.md | 22 ++++++++++ .../integration_tests/abnormal_state.json | 7 ++-- .../integration_tests/configured_catalog.json | 42 ++++++------------- .../integration_tests/invalid_config.json | 6 ++- .../source_rki_covid/source.py | 26 ------------ .../unit_tests/test_stream_germany.py | 4 +- 8 files changed, 58 insertions(+), 77 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-rki-covid/bootstrap.md diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml index 57289e2f271a7..149e163846675 100644 --- a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml @@ -7,24 +7,22 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] +# discovery: +# - config_path: "secrets/config.json" +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# empty_streams: [ "germany", "germany_age_groups" , "germany_history_cases"] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" # extra_fields: no # exact_order: no # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" +# incremental: # TODO +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# full_refresh: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md new file mode 100644 index 0000000000000..6ff79555089f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md @@ -0,0 +1,22 @@ +The Robert Koch-Institut - von Marlon Lückert) Covid-19 is [a REST based API](https://api.corona-zahlen.org/). +Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +## Cases In Germany Covid api stream +The basic entry stream is 'germany'. All other streams are extended version of base stream and passing parameters also result in sliced data. +For production, every developer application can view multiple streams. +[This endpoint](https://api.corona-zahlen.org/germany/germany/history/cases/:days) gets a list of cases in germany in perticular day. + +## Other streams +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/cases/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/incidence/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/deaths/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/recovered/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/frozen-incidence/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/hospitalization/1) \(Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany) \(Non-Incremental\) +* [This endpoint](https://api.corona-zahlen.org/germany/age-groups) \(Non-Incremental\) + + + +Incremental streams have required parameter days. Without passing days as parameter full-refresh happens. +As cursor field this connector uses "date". \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json index 52b0f2c2118f4..e4fc6f53e6f90 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -1,5 +1,6 @@ { - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" + "germany_history_cases": { + "date": "2022-04-04T00:00:00.000Z", + "cases": "123" } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index 480f747715e45..398776f762277 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -1,28 +1,10 @@ { "streams": [ - { - "stream": { - "name": "germany", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "germany_age_groups", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append_dedup" - }, { "stream": { "name": "germany_history_cases", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" @@ -30,8 +12,8 @@ { "stream": { "name": "german_history_incidence", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" @@ -39,8 +21,8 @@ { "stream": { "name": "german_history_deaths", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" @@ -48,8 +30,8 @@ { "stream": { "name": "german_history_recovered", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" @@ -57,8 +39,8 @@ { "stream": { "name": "german_history_frozen_incidence", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" @@ -66,8 +48,8 @@ { "stream": { "name": "german_history_hospitalization", - "json_schema": {"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]}, - "supported_sync_modes": ["incremental"] + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json index f3732995784f2..9e554b78843fc 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json @@ -1,3 +1,7 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "incidence_in_days": 100, + "deaths_in_days": 100, + "recovered_in_days": 100, + "frozen_incidence_in_days": 100, + "hospitalization_in_days":100 } diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 52dcec12a7d57..8725f0a39a69c 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -105,10 +105,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" @property @@ -129,7 +125,6 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: self._cursor_value = current_stream_state assert self._cursor_value == current_stream_state - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -157,10 +152,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ @@ -198,10 +189,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ @@ -237,10 +224,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ @@ -276,10 +259,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ @@ -315,10 +294,6 @@ def source_defined_cursor(self) -> bool: @property def cursor_field(self) -> str: - """ - date is cursor field in the stream. - :return str: The name of the cursor field. - """ return "date" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ @@ -361,7 +336,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ - Defining streams to run. :param config: A Mapping of the user input configuration as defined in the connector spec. """ diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py index 1695eec0c17e4..4f119e447f2b3 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py @@ -17,5 +17,5 @@ def patch_germany_class(mocker): def test_path(patch_germany_class): stream = Germany() - expected_params = {"path": "germany"} - assert stream.path() == expected_params.get("path") \ No newline at end of file + expected_params = {"path": "germany/"} + assert stream.path() == expected_params.get("path") From b5391db3aa05ca31d7145a99b7bb01e6418d5f78 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Tue, 5 Apr 2022 18:45:17 +0200 Subject: [PATCH 15/34] On path with master --- airbyte-integrations/connectors/source-rki-covid/bootstrap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md index 6ff79555089f6..8a4cbdedb71d6 100644 --- a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md +++ b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md @@ -1,4 +1,4 @@ -The Robert Koch-Institut - von Marlon Lückert) Covid-19 is [a REST based API](https://api.corona-zahlen.org/). +The (Robert Koch-Institut - von Marlon Lückert) Covid-19 is [a REST based API](https://api.corona-zahlen.org/). Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). ## Cases In Germany Covid api stream From 3eaa12957936f13abcf55b295187dbcb8844c617 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Fri, 8 Apr 2022 23:28:07 +0200 Subject: [PATCH 16/34] updated the changes for PR request. --- .../acceptance-test-config.yml | 26 +-- .../integration_tests/abnormal_state.json | 18 +- .../integration_tests/configured_catalog.json | 49 +---- .../schemas/german_history_deaths.json | 2 +- .../german_history_frozen_incidence.json | 2 +- .../german_history_hospitalization.json | 2 +- .../schemas/german_history_incidence.json | 2 +- .../schemas/german_history_recovered.json | 2 +- .../schemas/germany_age_groups.json | 2 +- .../schemas/germany_history_cases.json | 2 +- .../source_rki_covid/source.py | 177 +++++++++++++----- .../source_rki_covid/spec.json | 26 +-- main.py | 16 -- 13 files changed, 166 insertions(+), 160 deletions(-) delete mode 100644 main.py diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml index 149e163846675..ad3b3adbb56de 100644 --- a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml @@ -7,22 +7,22 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" -# discovery: -# - config_path: "secrets/config.json" -# basic_read: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# empty_streams: [ "germany", "germany_age_groups" , "germany_history_cases"] + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [ "germany", "germany_age_groups", "german_history_frozen_incidence"] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" # extra_fields: no # exact_order: no # extra_records: yes -# incremental: # TODO -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" -# full_refresh: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: # TODO + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json index e4fc6f53e6f90..6e684ea08dfde 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -1,6 +1,18 @@ { "germany_history_cases": { - "date": "2022-04-04T00:00:00.000Z", - "cases": "123" - } + "date": "2024-04-06T00:00:00.000Z" + }, + "german_history_incidence": { + "date": "2024-04-06T00:00:00.000Z" + }, + "german_history_deaths": { + "date": "2024-04-06T00:00:00.000Z" + }, + "german_history_recovered": { + "date": "2024-04-06T00:00:00.000Z" + }, + "german_history_hospitalization": { + "date": "2024-04-07T00:00:00.000Z" + }, + "german_history_frozen_incidence": {} } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index 398776f762277..4f3f407fe92df 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -4,52 +4,9 @@ "stream": { "name": "germany_history_cases", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "german_history_incidence", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "german_history_deaths", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "german_history_recovered", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "german_history_frozen_incidence", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" - }, - { - "stream": { - "name": "german_history_hospitalization", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json index 4e32e429e3f31..0ffd7db3e3756 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"deaths": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "deaths"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"deaths": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "deaths"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json index 12abaadb0a228..b8c20848a8777 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "object", "properties": {"abbreviation": {"type": "string"}, "name": {"type": "string"}, "history": {"type": "array", "items": {"type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]}}}, "required": ["abbreviation", "history", "name"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json index 0c11ab30f2c02..7bb17869e5808 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "fixedCases7Days": {"type": ["integer", "null"]}, "updatedCases7Days": {"type": ["integer", "null"]}, "adjustedLowerCases7Days": {"type": ["integer", "null"]}, "adjustedCases7Days": {"type": ["integer", "null"]}, "adjustedUpperCases7Days": {"type": ["integer", "null"]}, "fixedIncidence7Days": {"type": ["null", "number"]}, "updatedIncidence7Days": {"type": ["null", "number"]}, "adjustedLowerIncidence7Days": {"type": ["null", "number"]}, "adjustedIncidence7Days": {"type": ["null", "number"]}, "adjustedUpperIncidence7Days": {"type": ["null", "number"]}}, "required": ["adjustedCases7Days", "adjustedIncidence7Days", "adjustedLowerCases7Days", "adjustedLowerIncidence7Days", "adjustedUpperCases7Days", "adjustedUpperIncidence7Days", "cases7Days", "date", "fixedCases7Days", "fixedIncidence7Days", "incidence7Days", "updatedCases7Days", "updatedIncidence7Days"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "fixedCases7Days": {"type": "integer"}, "updatedCases7Days": {"type": "integer"}, "adjustedLowerCases7Days": {"type": "null"}, "adjustedCases7Days": {"type": "null"}, "adjustedUpperCases7Days": {"type": "null"}, "fixedIncidence7Days": {"type": "number"}, "updatedIncidence7Days": {"type": "number"}, "adjustedLowerIncidence7Days": {"type": "null"}, "adjustedIncidence7Days": {"type": "null"}, "adjustedUpperIncidence7Days": {"type": "null"}}, "required": ["adjustedCases7Days", "adjustedIncidence7Days", "adjustedLowerCases7Days", "adjustedLowerIncidence7Days", "adjustedUpperCases7Days", "adjustedUpperIncidence7Days", "cases7Days", "date", "fixedCases7Days", "fixedIncidence7Days", "incidence7Days", "updatedCases7Days", "updatedIncidence7Days"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json index ccdbf229b1626..b8c20848a8777 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json index 32b957fe7746e..4a4cb164cff6d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"recovered": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "recovered"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"recovered": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "recovered"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json index 43b8b39a82dea..c2b405883c93d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "object", "properties": {"A00-A04": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A05-A14": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A15-A34": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A35-A59": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "integer"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A60-A79": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A80+": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "integer"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}}, "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"A00-A04": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A05-A14": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A15-A34": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "integer"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A35-A59": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "integer"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A60-A79": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "integer"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A80+": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}}, "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json index fb9c6ecfd7d9b..9de6ab6dcf368 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json @@ -1 +1 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"data": {"type": "array", "items": {"type": "object", "properties": {"cases": {"type": "integer"}, "date": {"type": "string"}}, "required": ["cases", "date"]}}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["data", "meta"]} +{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "date": {"type": "string"}}, "required": ["cases", "date"]} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 8725f0a39a69c..386afcce23a22 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -58,6 +58,9 @@ class GermanyAgeGroups(RkiCovidStream): primary_key = None + # def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # return response.json().get("data") + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: @@ -96,8 +99,7 @@ class GermanyHistoryCases(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config - self._cursor_value = None + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -107,23 +109,25 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" - @property - def state(self) -> Mapping[str, Any]: - return {self.cursor_field: str(self._cursor_value)} - - @state.setter - def state(self, value: Mapping[str, Any]): - self._cursor_value = value[self.cursor_field] - - def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: - for record in super().read_records(*args, **kwargs): - current_stream_state = record.get(self.cursor_field) - if self._cursor_value: - latest_state = record.get(self.cursor_field) - self._cursor_value = max(self._cursor_value, latest_state) - yield record - self._cursor_value = current_stream_state - assert self._cursor_value == current_stream_state + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ + Mapping[str, Any]: + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -131,8 +135,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('cases_in_days'): - return "germany/history/cases/"+str(self.config.get('cases_in_days')) + if self.start_date: + return "germany/history/cases/"+str(self.date_to_int(self.start_date)) return "germany/history/cases/" @@ -144,7 +148,7 @@ class GermanHistoryIncidence(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -154,11 +158,25 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: if response.json().get("data"): @@ -168,8 +186,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('incidence_in_days'): - return "germany/history/incidence/"+str(self.config.get('incidence_in_days')) + if self.start_date: + return "germany/history/incidence/"+str(self.date_to_int(self.start_date)) return "germany/history/incidence/" @@ -181,7 +199,7 @@ class GermanHistoryDeaths(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -191,11 +209,25 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -203,8 +235,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('deaths_in_days'): - return "germany/history/deaths/"+str(self.config.get('deaths_in_days')) + if self.start_date: + return "germany/history/deaths/"+str(self.date_to_int(self.start_date)) return "germany/history/deaths/" @@ -216,7 +248,7 @@ class GermanHistoryRecovered(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -226,11 +258,25 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data") @@ -238,8 +284,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('recovered_in_days'): - return "germany/history/recovered/"+str(self.config.get('recovered_in_days')) + if self.start_date: + return "germany/history/recovered/"+str(self.date_to_int(self.start_date)) return "germany/history/recovered/" @@ -251,7 +297,7 @@ class GermanHistoryFrozenIncidence(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -261,11 +307,25 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return response.json().get("data").get("history") @@ -273,8 +333,8 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('frozen_incidence_in_days'): - return "germany/history/frozen-incidence/"+str(self.config.get('frozen_incidence_in_days')) + if self.start_date: + return "germany/history/frozen-incidence/"+str(self.date_to_int(self.start_date)) return "germany/history/frozen-incidence/" @@ -286,7 +346,7 @@ class GermanHistoryHospitalization(IncrementalRkiCovidStream): def __init__(self, config, **kwargs): super().__init__(**kwargs) - self.config = config + self.start_date = config.get("start_date") @property def source_defined_cursor(self) -> bool: @@ -296,21 +356,34 @@ def source_defined_cursor(self) -> bool: def cursor_field(self) -> str: return "date" + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + return diff.days + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ Mapping[str, Any]: - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - return {self.cursor_field: max(latest_state, current_state)} + if not current_stream_state: + current_stream_state = {self.cursor_field: self.start_date} + return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + + def read_records( + self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + records = super().read_records(stream_state=stream_state, **kwargs) + if stream_state: + for record in records: + if record[self.cursor_field] > stream_state.get(self.cursor_field): + yield record + else: + yield from records def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.logger.info(f"Cursor Field is = {self.cursor_field}") return response.json().get("data") def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - if self.config.get('hospitalization_in_days'): - return "germany/history/hospitalization/"+str(self.config.get('hospitalization_in_days')) + if self.start_date: + return "germany/history/hospitalization/"+str(self.date_to_int(self.start_date)) return "germany/history/hospitalization/" diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json index 64544cd268ead..548084289eb0d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -7,29 +7,9 @@ "required": [], "additionalProperties": false, "properties": { - "cases_in_days": { - "type": "integer", - "description": "Number of cases days in the past from today." - }, - "incidence_in_days": { - "type": "integer", - "description": "Number of incidence days in the past from today." - }, - "deaths_in_days": { - "type": "integer", - "description": "Number of deaths days in the past from today." - }, - "recovered_in_days": { - "type": "integer", - "description": "Number of recovered days in the past from today." - }, - "frozen_incidence_in_days": { - "type": "integer", - "description": "Number of frozen-incidence days in the past from today." - }, - "hospitalization_in_days": { - "type": "integer", - "description": "Number of hospitalization days in the past from today." + "start_date": { + "type": "string", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated." } } } diff --git a/main.py b/main.py deleted file mode 100644 index 94e3a87232a22..0000000000000 --- a/main.py +++ /dev/null @@ -1,16 +0,0 @@ -# This is a sample Python script. - -# Press ⌃R to execute it or replace it with your code. -# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings. - - -def print_hi(name): - # Use a breakpoint in the code line below to debug your script. - print(f'Hi, {name}') # Press ⌘F8 to toggle the breakpoint. - - -# Press the green button in the gutter to run the script. -if __name__ == '__main__': - print_hi('PyCharm') - -# See PyCharm help at https://www.jetbrains.com/help/pycharm/ From c37d84316dfdae7e4d418200f10b4390916f59bf Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Wed, 20 Apr 2022 15:11:31 +0200 Subject: [PATCH 17/34] changed file source-rki-covid/integration_tests/configured_catalog.json --- .../source-rki-covid/integration_tests/configured_catalog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index 4f3f407fe92df..8f50c8b8613dd 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -12,4 +12,4 @@ "destination_sync_mode": "append_dedup" } ] -} +} \ No newline at end of file From 1f259a8dd3eb964858bd911fe6f0dd086034d566 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Wed, 27 Apr 2022 16:24:55 +0200 Subject: [PATCH 18/34] corrected flake and blackformatting. Build gradel. --- .../integration_tests/acceptance.py | 1 - .../source_rki_covid/source.py | 72 +++++++++---------- .../unit_tests/test_cached_stream_state.py | 5 ++ .../test_incremental_germanhistorycases.py | 26 ++++--- .../test_incremental_germanhistorydeaths.py | 28 +++++--- ...ncremental_germanhistoryfrozenIncidence.py | 26 ++++--- ...ncremental_germanhistoryhospitalization.py | 28 +++++--- ...test_incremental_germanhistoryincidence.py | 28 +++++--- ...test_incremental_germanhistoryrecovered.py | 28 +++++--- .../unit_tests/test_stream_agegroup.py | 7 +- .../unit_tests/test_stream_germany.py | 5 +- 11 files changed, 148 insertions(+), 106 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py index c1f0803ff9a6e..0347f2a0b143d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py @@ -12,4 +12,3 @@ def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" yield - diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 386afcce23a22..b813bfd921d49 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -4,20 +4,18 @@ from abc import ABC +from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests -from datetime import datetime from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator # Basic full refresh stream class RkiCovidStream(HttpStream, ABC): - url_base = "https://api.corona-zahlen.org/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -111,16 +109,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -136,7 +134,7 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/cases/"+str(self.date_to_int(self.start_date)) + return "germany/history/cases/" + str(self.date_to_int(self.start_date)) return "germany/history/cases/" @@ -160,16 +158,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -187,7 +185,7 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/incidence/"+str(self.date_to_int(self.start_date)) + return "germany/history/incidence/" + str(self.date_to_int(self.start_date)) return "germany/history/incidence/" @@ -211,16 +209,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -236,7 +234,7 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/deaths/"+str(self.date_to_int(self.start_date)) + return "germany/history/deaths/" + str(self.date_to_int(self.start_date)) return "germany/history/deaths/" @@ -260,16 +258,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -285,7 +283,7 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/recovered/"+str(self.date_to_int(self.start_date)) + return "germany/history/recovered/" + str(self.date_to_int(self.start_date)) return "germany/history/recovered/" @@ -309,16 +307,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -334,7 +332,7 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/frozen-incidence/"+str(self.date_to_int(self.start_date)) + return "germany/history/frozen-incidence/" + str(self.date_to_int(self.start_date)) return "germany/history/frozen-incidence/" @@ -358,16 +356,16 @@ def cursor_field(self) -> str: def date_to_int(self, start_date) -> int: diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return 1 return diff.days - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> \ - Mapping[str, Any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: if not current_stream_state: current_stream_state = {self.cursor_field: self.start_date} return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - def read_records( - self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(stream_state=stream_state, **kwargs) if stream_state: for record in records: @@ -383,13 +381,12 @@ def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: if self.start_date: - return "germany/history/hospitalization/"+str(self.date_to_int(self.start_date)) + return "germany/history/hospitalization/" + str(self.date_to_int(self.start_date)) return "germany/history/hospitalization/" # Source class SourceRkiCovid(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: """ Testing connection availability for the connector. @@ -399,14 +396,13 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ try: - req = requests.get(RkiCovidStream.url_base+'germany') + req = requests.get(RkiCovidStream.url_base + "germany") if req.status_code == 200: return True, None return False, req.reason except Exception: return False, "There is a problem in source check connection." - def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. @@ -420,5 +416,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: GermanHistoryDeaths(config=config), GermanHistoryRecovered(config=config), GermanHistoryFrozenIncidence(config=config), - GermanHistoryHospitalization(config=config) - ] + GermanHistoryHospitalization(config=config), + ] diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py index e69de29bb2d1d..0f599765708a6 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_cached_stream_state.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +# diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py index ced7f2993301f..7604157b1d749 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py @@ -3,11 +3,12 @@ # +from datetime import datetime + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanyHistoryCases +from source_rki_covid.source import GermanyHistoryCases + @fixture def patch_incremental_german_history_cases(mocker): @@ -16,24 +17,31 @@ def patch_incremental_german_history_cases(mocker): def test_cursor_field(patch_incremental_german_history_cases): - config = {"cases_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanyHistoryCases(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_parse_response(patch_incremental_german_history_cases): - config = {"cases_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanyHistoryCases(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/cases/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/cases/1") expected_response = response.json().get("data") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_cases): - config = {"cases_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanyHistoryCases(config) - expected_stream_path = "germany/history/cases/"+str(config.get('cases_in_days')) + expected_stream_path = "germany/history/cases/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path @@ -41,4 +49,4 @@ def test_parse_without_cases(patch_incremental_german_history_cases): config = {} stream = GermanyHistoryCases(config) expected_stream_path = "germany/history/cases/" - assert stream.path() == expected_stream_path \ No newline at end of file + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py index 9382e46f5b8d5..e16ced3090177 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py @@ -3,11 +3,12 @@ # +from datetime import datetime, timedelta + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryDeaths +from source_rki_covid.source import GermanHistoryDeaths + @fixture def patch_incremental_german_history_deaths(mocker): @@ -16,14 +17,14 @@ def patch_incremental_german_history_deaths(mocker): def test_cursor_field(patch_incremental_german_history_deaths): - config = {"deaths_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryDeaths(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_german_history_deaths): - config = {"deaths_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryDeaths(config) d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} @@ -33,17 +34,24 @@ def test_get_updated_state(patch_incremental_german_history_deaths): def test_parse_response(patch_incremental_german_history_deaths): - config = {"deaths_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryDeaths(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/deaths/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/deaths/1") expected_response = response.json().get("data") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_deaths): - config = {"deaths_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryDeaths(config) - expected_stream_path = "germany/history/deaths/"+str(config.get('deaths_in_days')) + expected_stream_path = "germany/history/deaths/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path @@ -51,4 +59,4 @@ def test_parse_without_cases(patch_incremental_german_history_deaths): config = {} stream = GermanHistoryDeaths(config) expected_stream_path = "germany/history/deaths/" - assert stream.path() == expected_stream_path \ No newline at end of file + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py index 4d45e23984b80..d587ee89644e6 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py @@ -3,11 +3,12 @@ # +from datetime import datetime, timedelta + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryFrozenIncidence +from source_rki_covid.source import GermanHistoryFrozenIncidence + @fixture def patch_incremental_german_history_frozenInc(mocker): @@ -16,14 +17,14 @@ def patch_incremental_german_history_frozenInc(mocker): def test_cursor_field(patch_incremental_german_history_frozenInc): - config = {"frozen_incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryFrozenIncidence(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_german_history_frozenInc): - config = {"frozen_incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryFrozenIncidence(config) d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} @@ -33,17 +34,24 @@ def test_get_updated_state(patch_incremental_german_history_frozenInc): def test_parse_response(patch_incremental_german_history_frozenInc): - config = {"frozen_incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryFrozenIncidence(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/frozen-incidence/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/frozen-incidence/1") expected_response = response.json().get("data").get("history") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_frozenInc): - config = {"frozen_incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryFrozenIncidence(config) - expected_stream_path = "germany/history/frozen-incidence/"+str(config.get('frozen_incidence_in_days')) + expected_stream_path = "germany/history/frozen-incidence/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py index 3950cab7b442e..a4c2c0ae58cce 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py @@ -3,11 +3,12 @@ # +from datetime import datetime, timedelta + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryHospitalization +from source_rki_covid.source import GermanHistoryHospitalization + @fixture def patch_incremental_german_history_hospitalization(mocker): @@ -16,14 +17,14 @@ def patch_incremental_german_history_hospitalization(mocker): def test_cursor_field(patch_incremental_german_history_hospitalization): - config = {"hospitalization_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryHospitalization(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_german_history_hospitalization): - config = {"hospitalization_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryHospitalization(config) d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} @@ -33,17 +34,24 @@ def test_get_updated_state(patch_incremental_german_history_hospitalization): def test_parse_response(patch_incremental_german_history_hospitalization): - config = {"hospitalization_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryHospitalization(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/hospitalization/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/hospitalization/1") expected_response = response.json().get("data") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_hospitalization): - config = {"hospitalization_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryHospitalization(config) - expected_stream_path = "germany/history/hospitalization/"+str(config.get('hospitalization_in_days')) + expected_stream_path = "germany/history/hospitalization/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path @@ -51,4 +59,4 @@ def test_parse_without_cases(patch_incremental_german_history_hospitalization): config = {} stream = GermanHistoryHospitalization(config) expected_stream_path = "germany/history/hospitalization/" - assert stream.path() == expected_stream_path \ No newline at end of file + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py index 2f291a436fbe6..4d30ac970930f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py @@ -3,11 +3,12 @@ # +from datetime import datetime, timedelta + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryIncidence +from source_rki_covid.source import GermanHistoryIncidence + @fixture def patch_incremental_german_history_incidence(mocker): @@ -16,14 +17,14 @@ def patch_incremental_german_history_incidence(mocker): def test_cursor_field(patch_incremental_german_history_incidence): - config = {"incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryIncidence(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_german_history_incidence): - config = {"incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryIncidence(config) d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} @@ -33,17 +34,24 @@ def test_get_updated_state(patch_incremental_german_history_incidence): def test_parse_response(patch_incremental_german_history_incidence): - config = {"incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryIncidence(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/incidence/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/incidence/1") expected_response = response.json().get("data") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_incidence): - config = {"incidence_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryIncidence(config) - expected_stream_path = "germany/history/incidence/"+str(config.get('incidence_in_days')) + expected_stream_path = "germany/history/incidence/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path @@ -51,4 +59,4 @@ def test_parse_without_cases(patch_incremental_german_history_incidence): config = {} stream = GermanHistoryIncidence(config) expected_stream_path = "germany/history/incidence/" - assert stream.path() == expected_stream_path \ No newline at end of file + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py index 674adc5ba81a0..6bcc250c82389 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py @@ -3,11 +3,12 @@ # +from datetime import datetime, timedelta + import requests -from airbyte_cdk.models import SyncMode from pytest import fixture -from datetime import datetime, timedelta -from source_rki_covid.source import IncrementalRkiCovidStream, GermanHistoryRecovered +from source_rki_covid.source import GermanHistoryRecovered + @fixture def patch_incremental_german_history_recovered(mocker): @@ -16,14 +17,14 @@ def patch_incremental_german_history_recovered(mocker): def test_cursor_field(patch_incremental_german_history_recovered): - config = {"recovered_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryRecovered(config) expected_cursor_field = "date" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_german_history_recovered): - config = {"recovered_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryRecovered(config) d = datetime.date(datetime.today()) - timedelta(days=1) date = {stream.cursor_field: str(d)} @@ -33,17 +34,24 @@ def test_get_updated_state(patch_incremental_german_history_recovered): def test_parse_response(patch_incremental_german_history_recovered): - config = {"recovered_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryRecovered(config) - response = requests.get('https://api.corona-zahlen.org/germany/history/recovered/1') + response = requests.get("https://api.corona-zahlen.org/germany/history/recovered/1") expected_response = response.json().get("data") assert stream.parse_response(response) == expected_response +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days == 0: + return str(1) + return str(diff.days) + + def test_parse_with_cases(patch_incremental_german_history_recovered): - config = {"recovered_in_days": 2} + config = {"start_date": "2022-04-27"} stream = GermanHistoryRecovered(config) - expected_stream_path = "germany/history/recovered/"+str(config.get('recovered_in_days')) + expected_stream_path = "germany/history/recovered/" + check_diff(config.get("start_date")) assert stream.path() == expected_stream_path @@ -51,4 +59,4 @@ def test_parse_without_cases(patch_incremental_german_history_recovered): config = {} stream = GermanHistoryRecovered(config) expected_stream_path = "germany/history/recovered/" - assert stream.path() == expected_stream_path \ No newline at end of file + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py index 5e262c4d8f035..a669d9135777f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_agegroup.py @@ -2,11 +2,8 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from http import HTTPStatus -from unittest.mock import MagicMock - import pytest -from source_rki_covid.source import RkiCovidStream, GermanyAgeGroups +from source_rki_covid.source import GermanyAgeGroups @pytest.fixture @@ -18,4 +15,4 @@ def patch_age_group(mocker): def test_path(patch_age_group): stream = GermanyAgeGroups() expected_params = {"path": "germany/age-groups"} - assert stream.path() == expected_params.get("path") \ No newline at end of file + assert stream.path() == expected_params.get("path") diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py index 4f119e447f2b3..c3834e5f99207 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany.py @@ -2,11 +2,8 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from http import HTTPStatus -from unittest.mock import MagicMock - import pytest -from source_rki_covid.source import RkiCovidStream, Germany +from source_rki_covid.source import Germany @pytest.fixture From 286e5656200e5b1a343a168b4044ad1e48df6355 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Thu, 28 Apr 2022 11:48:55 +0200 Subject: [PATCH 19/34] Worked on the suggestions. --- .../connectors/source-rki-covid/bootstrap.md | 21 +- .../integration_tests/catalog.json | 39 -- .../integration_tests/configured_catalog.json | 73 ++++ .../integration_tests/invalid_config.json | 6 +- .../integration_tests/sample_config.json | 2 +- .../integration_tests/sample_state.json | 21 +- .../schemas/german_history_deaths.json | 17 +- .../german_history_frozen_incidence.json | 17 +- .../german_history_hospitalization.json | 61 ++- .../schemas/german_history_incidence.json | 17 +- .../schemas/german_history_recovered.json | 17 +- .../source_rki_covid/schemas/germany.json | 153 +++++++- .../schemas/germany_age_groups.json | 369 +++++++++++++++++- .../schemas/germany_history_cases.json | 17 +- .../source_rki_covid/source.py | 4 +- .../source_rki_covid/spec.json | 2 +- 16 files changed, 765 insertions(+), 71 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md index 8a4cbdedb71d6..5061ac55ce925 100644 --- a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md +++ b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md @@ -4,19 +4,18 @@ Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-de ## Cases In Germany Covid api stream The basic entry stream is 'germany'. All other streams are extended version of base stream and passing parameters also result in sliced data. For production, every developer application can view multiple streams. -[This endpoint](https://api.corona-zahlen.org/germany/germany/history/cases/:days) gets a list of cases in germany in perticular day. -## Other streams -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/cases/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/incidence/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/deaths/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/recovered/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/frozen-incidence/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/germany/history/hospitalization/1) \(Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany) \(Non-Incremental\) -* [This endpoint](https://api.corona-zahlen.org/germany/age-groups) \(Non-Incremental\) +## Endpoints +* [Provides covid cases and other information in Germany.](https://api.corona-zahlen.org/germany) \(Non-Incremental\ Entry-Stream) +* [Provides covid cases and other information in Germany, group by age.](https://api.corona-zahlen.org/germany/age-groups) \(Non-Incremental\) +* [Provides cases in Germany based on days.](https://api.corona-zahlen.org/germany/germany/history/cases/:days) \(Incremental\) +* [Provides incidence rate of covid in Germany based on days.](https://api.corona-zahlen.org/germany/germany/history/incidence/:days) \(Incremental\) +* [Provides death rate in Germany over days](https://api.corona-zahlen.org/germany/germany/history/deaths/:days) \(Incremental\) +* [Provides recovery rate in Germany over days.](https://api.corona-zahlen.org/germany/germany/history/recovered/:days) \(Incremental\) +* [Provides frozen incidence in Germany over days.](https://api.corona-zahlen.org/germany/germany/history/frozen-incidence/:days) \(Incremental\) +* [Provides hospitalization rate in Germany over days.](https://api.corona-zahlen.org/germany/germany/history/hospitalization/:days) \(Incremental\) -Incremental streams have required parameter days. Without passing days as parameter full-refresh happens. +Incremental streams have required parameter start-date. Without passing start-date as parameter full-refresh occurs. As cursor field this connector uses "date". \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json deleted file mode 100644 index 6799946a68514..0000000000000 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index 8f50c8b8613dd..ba258baf1d611 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -1,5 +1,23 @@ { "streams": [ + { + "stream": { + "name": "germany", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "germany_age_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, { "stream": { "name": "germany_history_cases", @@ -10,6 +28,61 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_incidence", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_deaths", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_recovered", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_frozen_incidence", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "german_history_hospitalization", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json index 9e554b78843fc..badf3b47a14e5 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/invalid_config.json @@ -1,7 +1,3 @@ { - "incidence_in_days": 100, - "deaths_in_days": 100, - "recovered_in_days": 100, - "frozen_incidence_in_days": 100, - "hospitalization_in_days":100 + "start_date": "20220426" } diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json index ecc4913b84c74..f82b6cc40f313 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "fix-me": "TODO" + "start_date": "2022-01-01" } diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json index 3587e579822d0..6a2719c2ca223 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json @@ -1,5 +1,18 @@ { - "todo-stream-name": { - "todo-field-name": "value" - } -} + "germany_history_cases": { + "date": "2024-01-01T00:00:00.000Z" + }, + "german_history_incidence": { + "date": "2024-01-01T00:00:00.000Z" + }, + "german_history_deaths": { + "date": "2024-01-01T00:00:00.000Z" + }, + "german_history_recovered": { + "date": "2024-01-01T00:00:00.000Z" + }, + "german_history_hospitalization": { + "date": "2024-01-01T00:00:00.000Z" + }, + "german_history_frozen_incidence": {} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json index 0ffd7db3e3756..1c5e2a1647bcb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json @@ -1 +1,16 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"deaths": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "deaths"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "deaths": { + "type": "integer" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "deaths" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json index b8c20848a8777..cf959be8a1f28 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json @@ -1 +1,16 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "weekIncidence" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json index 7bb17869e5808..18418b8043f8e 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json @@ -1 +1,60 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "fixedCases7Days": {"type": "integer"}, "updatedCases7Days": {"type": "integer"}, "adjustedLowerCases7Days": {"type": "null"}, "adjustedCases7Days": {"type": "null"}, "adjustedUpperCases7Days": {"type": "null"}, "fixedIncidence7Days": {"type": "number"}, "updatedIncidence7Days": {"type": "number"}, "adjustedLowerIncidence7Days": {"type": "null"}, "adjustedIncidence7Days": {"type": "null"}, "adjustedUpperIncidence7Days": {"type": "null"}}, "required": ["adjustedCases7Days", "adjustedIncidence7Days", "adjustedLowerCases7Days", "adjustedLowerIncidence7Days", "adjustedUpperCases7Days", "adjustedUpperIncidence7Days", "cases7Days", "date", "fixedCases7Days", "fixedIncidence7Days", "incidence7Days", "updatedCases7Days", "updatedIncidence7Days"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + }, + "fixedCases7Days": { + "type": "integer" + }, + "updatedCases7Days": { + "type": "integer" + }, + "adjustedLowerCases7Days": { + "type": "null" + }, + "adjustedCases7Days": { + "type": "null" + }, + "adjustedUpperCases7Days": { + "type": "null" + }, + "fixedIncidence7Days": { + "type": "number" + }, + "updatedIncidence7Days": { + "type": "number" + }, + "adjustedLowerIncidence7Days": { + "type": "null" + }, + "adjustedIncidence7Days": { + "type": "null" + }, + "adjustedUpperIncidence7Days": { + "type": "null" + } + }, + "required": [ + "adjustedCases7Days", + "adjustedIncidence7Days", + "adjustedLowerCases7Days", + "adjustedLowerIncidence7Days", + "adjustedUpperCases7Days", + "adjustedUpperIncidence7Days", + "cases7Days", + "date", + "fixedCases7Days", + "fixedIncidence7Days", + "incidence7Days", + "updatedCases7Days", + "updatedIncidence7Days" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json index b8c20848a8777..cf959be8a1f28 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json @@ -1 +1,16 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"weekIncidence": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "weekIncidence"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "weekIncidence" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json index 4a4cb164cff6d..df242cca3ed39 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json @@ -1 +1,16 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"recovered": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "recovered"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "recovered": { + "type": "integer" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "recovered" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json index cd838b986cb62..416e75d89832f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json @@ -1 +1,152 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}, "weekIncidence": {"type": "number"}, "casesPer100k": {"type": "number"}, "casesPerWeek": {"type": "integer"}, "delta": {"type": "object", "properties": {"cases": {"type": "integer"}, "deaths": {"type": "integer"}, "recovered": {"type": "integer"}}, "required": ["cases", "deaths", "recovered"]}, "r": {"type": "object", "properties": {"value": {"type": "number"}, "rValue4Days": {"type": "object", "properties": {"value": {"type": "number"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "rValue7Days": {"type": "object", "properties": {"value": {"type": "integer"}, "date": {"type": "string"}}, "required": ["date", "value"]}, "lastUpdate": {"type": "string"}}, "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"]}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}, "lastUpdate": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"]}, "meta": {"type": "object", "properties": {"source": {"type": "string"}, "contact": {"type": "string"}, "info": {"type": "string"}, "lastUpdate": {"type": "string"}, "lastCheckedForUpdate": {"type": "string"}}, "required": ["contact", "info", "lastCheckedForUpdate", "lastUpdate", "source"]}}, "required": ["cases", "casesPer100k", "casesPerWeek", "deaths", "delta", "hospitalization", "meta", "r", "recovered", "weekIncidence"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases": { + "type": "integer" + }, + "deaths": { + "type": "integer" + }, + "recovered": { + "type": "integer" + }, + "weekIncidence": { + "type": "number" + }, + "casesPer100k": { + "type": "number" + }, + "casesPerWeek": { + "type": "integer" + }, + "delta": { + "type": "object", + "properties": { + "cases": { + "type": "integer" + }, + "deaths": { + "type": "integer" + }, + "recovered": { + "type": "integer" + } + }, + "required": [ + "cases", + "deaths", + "recovered" + ] + }, + "r": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "rValue4Days": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "value" + ] + }, + "rValue7Days": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "date", + "value" + ] + }, + "lastUpdate": { + "type": "string" + } + }, + "required": [ + "lastUpdate", + "rValue4Days", + "rValue7Days", + "value" + ] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + }, + "lastUpdate": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days", + "lastUpdate" + ] + }, + "meta": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "contact": { + "type": "string" + }, + "info": { + "type": "string" + }, + "lastUpdate": { + "type": "string" + }, + "lastCheckedForUpdate": { + "type": "string" + } + }, + "required": [ + "contact", + "info", + "lastCheckedForUpdate", + "lastUpdate", + "source" + ] + } + }, + "required": [ + "cases", + "casesPer100k", + "casesPerWeek", + "deaths", + "delta", + "hospitalization", + "meta", + "r", + "recovered", + "weekIncidence" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json index c2b405883c93d..8f623aae32fd7 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -1 +1,368 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"A00-A04": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A05-A14": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A15-A34": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "integer"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A35-A59": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "integer"}, "deathsMalePer100k": {"type": "integer"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A60-A79": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "integer"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}, "A80+": {"type": "object", "properties": {"casesMale": {"type": "integer"}, "casesFemale": {"type": "integer"}, "deathsMale": {"type": "integer"}, "deathsFemale": {"type": "integer"}, "casesMalePer100k": {"type": "number"}, "casesFemalePer100k": {"type": "number"}, "deathsMalePer100k": {"type": "number"}, "deathsFemalePer100k": {"type": "number"}, "hospitalization": {"type": "object", "properties": {"cases7Days": {"type": "integer"}, "incidence7Days": {"type": "number"}, "date": {"type": "string"}}, "required": ["cases7Days", "date", "incidence7Days"]}}, "required": ["casesFemale", "casesFemalePer100k", "casesMale", "casesMalePer100k", "deathsFemale", "deathsFemalePer100k", "deathsMale", "deathsMalePer100k", "hospitalization"]}}, "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "A00-A04": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "integer" + }, + "casesFemalePer100k": { + "type": "number" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A05-A14": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "number" + }, + "casesFemalePer100k": { + "type": "number" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "integer" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A15-A34": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "number" + }, + "casesFemalePer100k": { + "type": "number" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A35-A59": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "integer" + }, + "casesFemalePer100k": { + "type": "number" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A60-A79": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "number" + }, + "casesFemalePer100k": { + "type": "number" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A80+": { + "type": "object", + "properties": { + "casesMale": { + "type": "integer" + }, + "casesFemale": { + "type": "integer" + }, + "deathsMale": { + "type": "integer" + }, + "deathsFemale": { + "type": "integer" + }, + "casesMalePer100k": { + "type": "number" + }, + "casesFemalePer100k": { + "type": "integer" + }, + "deathsMalePer100k": { + "type": "number" + }, + "deathsFemalePer100k": { + "type": "number" + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": "number" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases7Days", + "date", + "incidence7Days" + ] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + } + }, + "required": [ + "A00-A04", + "A05-A14", + "A15-A34", + "A35-A59", + "A60-A79", + "A80+" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json index 9de6ab6dcf368..9d2ad7bc4f9fb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json @@ -1 +1,16 @@ -{"$schema": "http://json-schema.org/schema#", "type": "object", "properties": {"cases": {"type": "integer"}, "date": {"type": "string"}}, "required": ["cases", "date"]} +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases": { + "type": "integer" + }, + "date": { + "type": "string" + } + }, + "required": [ + "cases", + "date" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index b813bfd921d49..956fc435817d4 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -56,8 +56,8 @@ class GermanyAgeGroups(RkiCovidStream): primary_key = None - # def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # return response.json().get("data") + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield response.json().get("data") def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json index 548084289eb0d..ed110aeac1f28 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -9,7 +9,7 @@ "properties": { "start_date": { "type": "string", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated." + "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated." } } } From 3daf8ddba04cb0aec72ffd1a3e1ff19c4be66f80 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 9 May 2022 10:15:04 +0200 Subject: [PATCH 20/34] source_rki_covid/schemas/germany_age_groups.json --- .../source_rki_covid/schemas/germany_age_groups.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json index 8f623aae32fd7..3e32308d224f4 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -95,7 +95,7 @@ "type": "integer" }, "incidence7Days": { - "type": "integer" + "type": "number" }, "date": { "type": "string" @@ -139,7 +139,7 @@ "type": "number" }, "casesFemalePer100k": { - "type": "number" + "type": "integer" }, "deathsMalePer100k": { "type": "number" @@ -195,7 +195,7 @@ "type": "integer" }, "casesMalePer100k": { - "type": "integer" + "type": "number" }, "casesFemalePer100k": { "type": "number" @@ -204,7 +204,7 @@ "type": "number" }, "deathsFemalePer100k": { - "type": "number" + "type": "integer" }, "hospitalization": { "type": "object", @@ -316,7 +316,7 @@ "type": "number" }, "casesFemalePer100k": { - "type": "integer" + "type": "number" }, "deathsMalePer100k": { "type": "number" From 904e3835c415e0cfcfa54f203d8986c0c0ce4f4f Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Mon, 9 May 2022 15:05:08 +0200 Subject: [PATCH 21/34] uodated abnormal_state.json --- .../source-rki-covid/integration_tests/abnormal_state.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json index 6e684ea08dfde..9da47366aa930 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -14,5 +14,7 @@ "german_history_hospitalization": { "date": "2024-04-07T00:00:00.000Z" }, - "german_history_frozen_incidence": {} + "german_history_frozen_incidence": { + "date": "2024-01-01T00:00:00.000Z" + } } \ No newline at end of file From dccea84dbe1801ce6e2725a01fc0e3241be12df1 Mon Sep 17 00:00:00 2001 From: "zawar.khan" Date: Tue, 17 May 2022 17:06:58 +0200 Subject: [PATCH 22/34] updated the schemas for german age groups and history hospitalization. --- .../german_history_hospitalization.json | 24 ++-- .../schemas/germany_age_groups.json | 120 +++++++++--------- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json index 18418b8043f8e..e459c2227e4c9 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json @@ -6,40 +6,40 @@ "type": "integer" }, "incidence7Days": { - "type": "number" + "type": ["number", "null"] }, "date": { - "type": "string" + "type": ["string", "null"] }, "fixedCases7Days": { - "type": "integer" + "type": ["integer", "null"] }, "updatedCases7Days": { - "type": "integer" + "type": ["integer", "null"] }, "adjustedLowerCases7Days": { - "type": "null" + "type": ["integer", "null"] }, "adjustedCases7Days": { - "type": "null" + "type": ["integer", "null"] }, "adjustedUpperCases7Days": { - "type": "null" + "type": ["integer", "null"] }, "fixedIncidence7Days": { - "type": "number" + "type": ["number", "null"] }, "updatedIncidence7Days": { - "type": "number" + "type": ["number", "null"] }, "adjustedLowerIncidence7Days": { - "type": "null" + "type": ["number", "null"] }, "adjustedIncidence7Days": { - "type": "null" + "type": ["number", "null"] }, "adjustedUpperIncidence7Days": { - "type": "null" + "type": ["number", "null"] } }, "required": [ diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json index 3e32308d224f4..6c2f9697add3d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -6,37 +6,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" @@ -65,37 +65,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" @@ -124,37 +124,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" @@ -183,37 +183,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "integer" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" @@ -242,37 +242,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" @@ -301,37 +301,37 @@ "type": "object", "properties": { "casesMale": { - "type": "integer" + "type": ["number", "integer"] }, "casesFemale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsMale": { - "type": "integer" + "type": ["number", "integer"] }, "deathsFemale": { - "type": "integer" + "type": ["number", "integer"] }, "casesMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "casesFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsMalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "deathsFemalePer100k": { - "type": "number" + "type": ["number", "integer"] }, "hospitalization": { "type": "object", "properties": { "cases7Days": { - "type": "integer" + "type": ["number", "integer"] }, "incidence7Days": { - "type": "number" + "type": ["number", "integer"] }, "date": { "type": "string" From e3519874ddf8363510ab0b43121b524f87a0e6a9 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 16:02:16 -0300 Subject: [PATCH 23/34] correct dockerfile and update airbyte_cdk version --- .../connectors/source-rki-covid/Dockerfile | 3 -- .../integration_tests/configured_catalog.json | 28 ++++++++++--------- .../connectors/source-rki-covid/setup.py | 8 ++---- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index 3376be9a98ad5..3471c6696ea63 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -24,9 +24,6 @@ COPY --from=builder /install /usr/local COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - # copy payload code only COPY main.py ./ COPY source_rki_covid ./source_rki_covid diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index ba258baf1d611..3ada28f227e00 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -7,7 +7,7 @@ "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append_dedup" + "destination_sync_mode": "overwrite" }, { "stream": { @@ -16,7 +16,7 @@ "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "append_dedup" + "destination_sync_mode": "overwrite" }, { "stream": { @@ -27,7 +27,8 @@ "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "cursor_field": ["date"], + "destination_sync_mode": "append" }, { "stream": { @@ -37,8 +38,8 @@ "source_defined_cursor": false, "default_cursor_field": ["date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -49,7 +50,8 @@ "default_cursor_field": ["date"] }, "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "cursor_field": ["date"], + "destination_sync_mode": "append" }, { "stream": { @@ -59,8 +61,8 @@ "source_defined_cursor": false, "default_cursor_field": ["date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -70,8 +72,8 @@ "source_defined_cursor": false, "default_cursor_field": ["date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -81,8 +83,8 @@ "source_defined_cursor": false, "default_cursor_field": ["date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py index 0d45f842d3e0a..c64e33f3442f5 100644 --- a/airbyte-integrations/connectors/source-rki-covid/setup.py +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -6,14 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "source-acceptance-test", -] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", "airbyte-cdk"] setup( name="source_rki_covid", From 562906e25ee0579ca4ec0b16a721c0160c4ebf24 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 16:10:16 -0300 Subject: [PATCH 24/34] run format --- .../RedshiftS3StagingSqlOperations.java | 26 +-- .../operations/RedshiftSqlOperations.java | 4 +- .../operations/RedshiftSqlOperationsTest.java | 153 +++++++++--------- .../integration_tests/configured_catalog.json | 5 +- .../unit_tests/conftest.py | 4 +- .../unit_tests/unit_test.py | 15 +- .../src/test/resources/expected_spec.json | 2 +- .../source-mssql/src/main/resources/spec.json | 2 +- .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/sample_state.json | 2 +- .../schemas/german_history_deaths.json | 7 +- .../german_history_frozen_incidence.json | 7 +- .../german_history_hospitalization.json | 2 +- .../schemas/german_history_incidence.json | 7 +- .../schemas/german_history_recovered.json | 7 +- .../source_rki_covid/schemas/germany.json | 32 +--- .../schemas/germany_age_groups.json | 47 +----- .../schemas/germany_history_cases.json | 7 +- .../source_scaffold_source_http/spec.yaml | 2 +- .../source_zendesk_chat/streams.py | 9 +- 20 files changed, 134 insertions(+), 208 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index 1029f802026c9..454f6e8939b39 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -32,8 +32,8 @@ public class RedshiftS3StagingSqlOperations extends RedshiftSqlOperations implem private final ObjectMapper objectMapper; public RedshiftS3StagingSqlOperations(NamingConventionTransformer nameTransformer, - AmazonS3 s3Client, - S3DestinationConfig s3Config) { + AmazonS3 s3Client, + S3DestinationConfig s3Config) { this.nameTransformer = nameTransformer; this.s3StorageOperations = new S3StorageOperations(nameTransformer, s3Client, s3Config); this.s3Config = s3Config; @@ -81,11 +81,11 @@ private String putManifest(final String manifestContents, String stagingPath) { @Override public void copyIntoTmpTableFromStage(JdbcDatabase database, - String stageName, - String stagingPath, - List stagedFiles, - String dstTableName, - String schemaName) + String stageName, + String stagingPath, + List stagedFiles, + String dstTableName, + String schemaName) throws Exception { LOGGER.info("Starting copy to tmp table from stage: {} in destination from stage: {}, schema: {}, .", dstTableName, stagingPath, schemaName); final var possibleManifest = Optional.ofNullable(createManifest(stagedFiles, stagingPath)); @@ -101,12 +101,12 @@ private void executeCopy(final String manifestPath, JdbcDatabase db, String sche final S3AccessKeyCredentialConfig credentialConfig = (S3AccessKeyCredentialConfig) s3Config.getS3CredentialConfig(); final var copyQuery = String.format( """ - COPY %s.%s FROM '%s' - CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s' - CSV GZIP - REGION '%s' TIMEFORMAT 'auto' - STATUPDATE OFF - MANIFEST;""", + COPY %s.%s FROM '%s' + CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s' + CSV GZIP + REGION '%s' TIMEFORMAT 'auto' + STATUPDATE OFF + MANIFEST;""", schemaName, tmpTableName, getFullS3Path(s3Config.getBucketName(), manifestPath), diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index fbda53d506b11..fbbe996dd2638 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -57,9 +57,7 @@ having count(*) = 3) ALTER TABLE %1$s RENAME %3$s_reserve to %3$s; """; - - public RedshiftSqlOperations() { - } + public RedshiftSqlOperations() {} @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java index 0003ddd5eae45..2b2936ceb7b76 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java @@ -6,91 +6,88 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.util.Random; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; - +import io.airbyte.commons.json.Jsons; +import java.util.Random; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import io.airbyte.commons.json.Jsons; - @DisplayName("RedshiftSqlOperations") public class RedshiftSqlOperationsTest { - private static final Random RANDOM = new Random(); - - private String generateBigString(final int addExtraCharacters) { - final int length = RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE + addExtraCharacters; - return RANDOM - .ints('a', 'z' + 1) - .limit(length) - .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - .toString(); - } - - @Test - @DisplayName("isValidData should return true for valid data") - public void isValidDataForValid() { - JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() - .put("id", 3) - .put("currency", generateBigString(0)) - .put("date", "2020-10-10T00:00:00Z") - .put("HKD", 10.5) - .put("NZD", 1.14) - .build()); - - RedshiftSqlOperations uut = new RedshiftSqlOperations(); - boolean isValid = uut.isValidData(testNode); - assertEquals(true, isValid); - } - - @Test - @DisplayName("isValidData should return false for invalid data - string too long") - public void isValidDataForInvalidNode() { - JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() - .put("id", 3) - .put("currency", generateBigString(1)) - .put("date", "2020-10-10T00:00:00Z") - .put("HKD", 10.5) - .put("NZD", 1.14) - .build()); - - RedshiftSqlOperations uut = new RedshiftSqlOperations(); - boolean isValid = uut.isValidData(testNode); - assertEquals(false, isValid); - } - - @Test - @DisplayName("isValidData should return false for invalid data - total object too big") - public void isValidDataForInvalidObject() { - JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() - .put("key1", generateBigString(-1)) - .put("key2", generateBigString(-1)) - .put("key3", generateBigString(-1)) - .put("key4", generateBigString(-1)) - .put("key5", generateBigString(-1)) - .put("key6", generateBigString(-1)) - .put("key7", generateBigString(-1)) - .put("key8", generateBigString(-1)) - .put("key9", generateBigString(-1)) - .put("key10", generateBigString(-1)) - .put("key11", generateBigString(-1)) - .put("key12", generateBigString(-1)) - .put("key13", generateBigString(-1)) - .put("key14", generateBigString(-1)) - .put("key15", generateBigString(-1)) - .put("key16", generateBigString(-1)) - .put("key17", generateBigString(-1)) - .put("key18", generateBigString(-1)) - .put("key19", generateBigString(-1)) - .put("key20", generateBigString(-1)) - .build()); - - RedshiftSqlOperations uut = new RedshiftSqlOperations(); - boolean isValid = uut.isValidData(testNode); - assertEquals(false, isValid); - } + private static final Random RANDOM = new Random(); + + private String generateBigString(final int addExtraCharacters) { + final int length = RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE + addExtraCharacters; + return RANDOM + .ints('a', 'z' + 1) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + + @Test + @DisplayName("isValidData should return true for valid data") + public void isValidDataForValid() { + JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() + .put("id", 3) + .put("currency", generateBigString(0)) + .put("date", "2020-10-10T00:00:00Z") + .put("HKD", 10.5) + .put("NZD", 1.14) + .build()); + + RedshiftSqlOperations uut = new RedshiftSqlOperations(); + boolean isValid = uut.isValidData(testNode); + assertEquals(true, isValid); + } + + @Test + @DisplayName("isValidData should return false for invalid data - string too long") + public void isValidDataForInvalidNode() { + JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() + .put("id", 3) + .put("currency", generateBigString(1)) + .put("date", "2020-10-10T00:00:00Z") + .put("HKD", 10.5) + .put("NZD", 1.14) + .build()); + + RedshiftSqlOperations uut = new RedshiftSqlOperations(); + boolean isValid = uut.isValidData(testNode); + assertEquals(false, isValid); + } + + @Test + @DisplayName("isValidData should return false for invalid data - total object too big") + public void isValidDataForInvalidObject() { + JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() + .put("key1", generateBigString(-1)) + .put("key2", generateBigString(-1)) + .put("key3", generateBigString(-1)) + .put("key4", generateBigString(-1)) + .put("key5", generateBigString(-1)) + .put("key6", generateBigString(-1)) + .put("key7", generateBigString(-1)) + .put("key8", generateBigString(-1)) + .put("key9", generateBigString(-1)) + .put("key10", generateBigString(-1)) + .put("key11", generateBigString(-1)) + .put("key12", generateBigString(-1)) + .put("key13", generateBigString(-1)) + .put("key14", generateBigString(-1)) + .put("key15", generateBigString(-1)) + .put("key16", generateBigString(-1)) + .put("key17", generateBigString(-1)) + .put("key18", generateBigString(-1)) + .put("key19", generateBigString(-1)) + .put("key20", generateBigString(-1)) + .build()); + + RedshiftSqlOperations uut = new RedshiftSqlOperations(); + boolean isValid = uut.isValidData(testNode); + assertEquals(false, isValid); + } } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index 6c10f506f3044..0f1b89da1c0ef 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -113,10 +113,7 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [ - ["ad_group_ad.ad.id"], - ["segments.date"] - ] + "primary_key": [["ad_group_ad.ad.id"], ["segments.date"]] }, { "stream": { diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py index 7f34bec9b24b8..f35f58b91492b 100644 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py @@ -17,8 +17,8 @@ def config_fixture(requests_mock): "auth_type": "Client", "client_id": "client_id", "client_secret": "client_secret", - "refresh_token": "refresh_token" - } + "refresh_token": "refresh_token", + }, } return config diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 69dd72fa7f0e9..df98742a688ee 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -8,10 +8,10 @@ import pytest from airbyte_cdk.models import SyncMode -from source_google_search_console.streams import ROW_LIMIT, SearchAnalyticsByDate, GoogleSearchConsole from source_google_search_console.source import SourceGoogleSearchConsole +from source_google_search_console.streams import ROW_LIMIT, GoogleSearchConsole, SearchAnalyticsByDate -logger = logging.getLogger('airbyte') +logger = logging.getLogger("airbyte") class MockResponse: @@ -116,12 +116,9 @@ def test_updated_state(): [ ( GoogleSearchConsole, - {'keys': ['keys']}, + {"keys": ["keys"]}, ), - ( - SearchAnalyticsByDate, - {"date": "keys", "search_type": "web", "site_url": "https://domain1.com"} - ) + (SearchAnalyticsByDate, {"date": "keys", "search_type": "web", "site_url": "https://domain1.com"}), ], ) @patch.multiple(GoogleSearchConsole, __abstractmethods__=set()) @@ -171,6 +168,8 @@ def test_streams(config): def test_get_start_date(): stream = SearchAnalyticsByDate(None, ["https://domain1.com", "https://domain2.com"], "2021-09-01", "2021-09-07") date = "2021-09-07" - state_date = stream._get_start_date(stream_state={"https://domain1.com": {"web": {"date": date}}}, site_url="https://domain1.com", search_type="web") + state_date = stream._get_start_date( + stream_state={"https://domain1.com": {"web": {"date": date}}}, site_url="https://domain1.com", search_type="web" + ) assert date == str(state_date) diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json index d3b48e6c3fd94..e78eb4b091cf3 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json @@ -124,7 +124,7 @@ "const": "CDC", "enum": ["CDC"], "default": "CDC", - "order":0 + "order": 0 }, "data_to_sync": { "title": "Data to Sync", diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index eb650b781a447..58421140a88dc 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -138,7 +138,7 @@ "const": "CDC", "enum": ["CDC"], "default": "CDC", - "order":0 + "order": 0 }, "data_to_sync": { "title": "Data to Sync", diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json index 9da47366aa930..f78d22b27ce40 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -17,4 +17,4 @@ "german_history_frozen_incidence": { "date": "2024-01-01T00:00:00.000Z" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json index 6a2719c2ca223..937df032816cb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/sample_state.json @@ -15,4 +15,4 @@ "date": "2024-01-01T00:00:00.000Z" }, "german_history_frozen_incidence": {} -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json index 1c5e2a1647bcb..a5f0eb9fd58d0 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_deaths.json @@ -9,8 +9,5 @@ "type": "string" } }, - "required": [ - "date", - "deaths" - ] -} \ No newline at end of file + "required": ["date", "deaths"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json index cf959be8a1f28..092c761412741 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_frozen_incidence.json @@ -9,8 +9,5 @@ "type": "string" } }, - "required": [ - "date", - "weekIncidence" - ] -} \ No newline at end of file + "required": ["date", "weekIncidence"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json index e459c2227e4c9..2af5e74b4ec0f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_hospitalization.json @@ -57,4 +57,4 @@ "updatedCases7Days", "updatedIncidence7Days" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json index cf959be8a1f28..092c761412741 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_incidence.json @@ -9,8 +9,5 @@ "type": "string" } }, - "required": [ - "date", - "weekIncidence" - ] -} \ No newline at end of file + "required": ["date", "weekIncidence"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json index df242cca3ed39..2e3015740a70c 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/german_history_recovered.json @@ -9,8 +9,5 @@ "type": "string" } }, - "required": [ - "date", - "recovered" - ] -} \ No newline at end of file + "required": ["date", "recovered"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json index 416e75d89832f..3cd32ad514842 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany.json @@ -33,11 +33,7 @@ "type": "integer" } }, - "required": [ - "cases", - "deaths", - "recovered" - ] + "required": ["cases", "deaths", "recovered"] }, "r": { "type": "object", @@ -55,10 +51,7 @@ "type": "string" } }, - "required": [ - "date", - "value" - ] + "required": ["date", "value"] }, "rValue7Days": { "type": "object", @@ -70,21 +63,13 @@ "type": "string" } }, - "required": [ - "date", - "value" - ] + "required": ["date", "value"] }, "lastUpdate": { "type": "string" } }, - "required": [ - "lastUpdate", - "rValue4Days", - "rValue7Days", - "value" - ] + "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"] }, "hospitalization": { "type": "object", @@ -102,12 +87,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days", - "lastUpdate" - ] + "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"] }, "meta": { "type": "object", @@ -149,4 +129,4 @@ "recovered", "weekIncidence" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json index 6c2f9697add3d..4b61518a0a673 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_age_groups.json @@ -42,11 +42,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -101,11 +97,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -160,11 +152,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -219,11 +207,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -278,11 +262,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -337,11 +317,7 @@ "type": "string" } }, - "required": [ - "cases7Days", - "date", - "incidence7Days" - ] + "required": ["cases7Days", "date", "incidence7Days"] } }, "required": [ @@ -357,12 +333,5 @@ ] } }, - "required": [ - "A00-A04", - "A05-A14", - "A15-A34", - "A35-A59", - "A60-A79", - "A80+" - ] -} \ No newline at end of file + "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json index 9d2ad7bc4f9fb..a13bb77774de2 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_history_cases.json @@ -9,8 +9,5 @@ "type": "string" } }, - "required": [ - "cases", - "date" - ] -} \ No newline at end of file + "required": ["cases", "date"] +} diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/spec.yaml b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/spec.yaml index f1c28b63d2925..99ff36cb6842f 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/spec.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/source_scaffold_source_http/spec.yaml @@ -7,7 +7,7 @@ connectionSpecification: - TODO additionalProperties: false properties: - # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': + # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': TODO: type: string description: describe me diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py index a01aba5efa927..cedf30980e21c 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py @@ -22,12 +22,13 @@ class Stream(HttpStream, ABC): limit = 100 def request_kwargs( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> Mapping[str, Any]: - return { - "timeout": 60 - } + return {"timeout": 60} def backoff_time(self, response: requests.Response) -> Optional[float]: delay_time = response.headers.get("Retry-After") From 83827e6091830e86f71026e1b29cfac805a9b730 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 16:41:27 -0300 Subject: [PATCH 25/34] update python version --- airbyte-integrations/connectors/source-rki-covid/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index 3471c6696ea63..1a853e0f30e53 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.11-alpine3.14 as base +FROM python:3.9-slim as base # build and load all requirements FROM base as builder From 8c446765da9958a3507ed91058e011e0a2316d8c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 17:01:24 -0300 Subject: [PATCH 26/34] correct dockerfile build --- .../connectors/source-rki-covid/Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index 1a853e0f30e53..bc0daab7021b5 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -1,14 +1,12 @@ -FROM python:3.9-slim as base +FROM python:3.9.11-slim as base # build and load all requirements FROM base as builder WORKDIR /airbyte/integration_code # upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* COPY setup.py ./ # install necessary packages to a temporary folder From 5c7f8d41dac9d1f58d5151da56d662f079a68ba6 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 17:35:25 -0300 Subject: [PATCH 27/34] add source in seed --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0f7549d539573..17275d76207d5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -776,6 +776,13 @@ icon: retently.svg sourceType: api releaseStage: alpha +- name: RKI Covid + sourceDefinitionId: db04ecd1-42e7-4115-9cec-95812905c626 + dockerRepository: airbyte/source-rki-covid + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/rki-covid + sourceType: api + releaseStage: alpha - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 From 9ea2a53550430a5a3c77e8b821d1544ab75389dc Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Thu, 26 May 2022 17:44:35 -0300 Subject: [PATCH 28/34] update uuid for rki-covid source --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 17275d76207d5..17eac54453bfd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -777,7 +777,7 @@ sourceType: api releaseStage: alpha - name: RKI Covid - sourceDefinitionId: db04ecd1-42e7-4115-9cec-95812905c626 + sourceDefinitionId: d78e5de0-aa44-4744-aa4f-74c818ccfe19 dockerRepository: airbyte/source-rki-covid dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/rki-covid From f3b23ab67ed851c5397a38f6728c75dc90e7886e Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 27 May 2022 16:25:44 -0300 Subject: [PATCH 29/34] change docker --- .../connectors/source-rki-covid/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index bc0daab7021b5..08c724861ce10 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -1,12 +1,11 @@ -FROM python:3.9.11-slim as base +FROM python:3.9.11-alpine3.15 as base -# build and load all requirements +# build and load all requirements FROM base as builder WORKDIR /airbyte/integration_code # upgrade pip to the latest version -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +RUN apk --no-cache upgrade && pip install --upgrade pip COPY setup.py ./ # install necessary packages to a temporary folder From 857dc42474fb4768d65b35c4b129b36970c1ad10 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 27 May 2022 16:45:17 -0300 Subject: [PATCH 30/34] add bash --- airbyte-integrations/connectors/source-rki-covid/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index 08c724861ce10..9aa59a76ffb7a 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -5,7 +5,9 @@ FROM base as builder WORKDIR /airbyte/integration_code # upgrade pip to the latest version -RUN apk --no-cache upgrade && pip install --upgrade pip +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base COPY setup.py ./ # install necessary packages to a temporary folder From 5e36611cb4942a62f9cafe770c717fb46e93faf9 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 27 May 2022 20:04:53 +0000 Subject: [PATCH 31/34] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 22bbe8ce76bb2..9e43f34909b07 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7148,6 +7148,23 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-rki-covid:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Rki Covid Spec" + type: "object" + required: [] + additionalProperties: false + properties: + start_date: + type: "string" + description: "UTC date in the format 2017-01-25. Any data before this date\ + \ will not be replicated." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-s3:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/s3" From 75e5c3398c3fb5efc035045f40278fa49ee28feb Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 27 May 2022 20:11:12 -0300 Subject: [PATCH 32/34] run seed file --- .../src/main/resources/seed/source_specs.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 22bbe8ce76bb2..9e43f34909b07 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7148,6 +7148,23 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-rki-covid:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Rki Covid Spec" + type: "object" + required: [] + additionalProperties: false + properties: + start_date: + type: "string" + description: "UTC date in the format 2017-01-25. Any data before this date\ + \ will not be replicated." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-s3:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/s3" From ef8f98a8570dc781956b65ae61626465b7990b78 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 30 May 2022 09:32:14 -0300 Subject: [PATCH 33/34] correct doc --- .../connectors/source-rki-covid/Dockerfile | 2 +- .../source_rki_covid/spec.json | 10 ++-- docs/integrations/sources/rki-covid.md | 57 +++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 docs/integrations/sources/rki-covid.md diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile index 9aa59a76ffb7a..f226dc7cd918f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/Dockerfile +++ b/airbyte-integrations/connectors/source-rki-covid/Dockerfile @@ -30,5 +30,5 @@ COPY source_rki_covid ./source_rki_covid ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-rki-covid diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json index ed110aeac1f28..eac69f9467bd2 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json @@ -1,15 +1,17 @@ { - "documentationUrl": "https://docsurl.com", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/rki-covid", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Rki Covid Spec", + "title": "RKI Covid Spec", "type": "object", - "required": [], + "required": ["start_date"], "additionalProperties": false, "properties": { "start_date": { "type": "string", - "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated." + "title": "Start Date", + "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated.", + "order": 1 } } } diff --git a/docs/integrations/sources/rki-covid.md b/docs/integrations/sources/rki-covid.md new file mode 100644 index 0000000000000..f887e22acde67 --- /dev/null +++ b/docs/integrations/sources/rki-covid.md @@ -0,0 +1,57 @@ +# Robert Koch-Institut Covid + +## Sync overview + +This source can sync data for the [Robert Koch-Institut Covid API](https://api.corona-zahlen.org/). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +### Output schema + +This Source is capable of syncing the following core Streams (only for Germany cases): + +* Germany +* Germany by age and groups +* Germany cases by days +* Germany incidences by days +* Germany deaths by days +* Germany recovered by days +* Germany frozen-incidence by days +* Germany hospitalization by days + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `integer` | `integer` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | + +### Performance considerations + +The RKI Covid connector should not run into RKI Covid API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Start Date + +### Setup guide + +Select start date + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.1 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Fix docs | +| 0.1.0 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Initial Release | From da459e4721aee6b71d60c6acaa0a395d04caf783 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 30 May 2022 12:51:07 +0000 Subject: [PATCH 34/34] auto-bump connector version --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1ce4e64519dcc..fcd79be087f8d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -779,7 +779,7 @@ - name: RKI Covid sourceDefinitionId: d78e5de0-aa44-4744-aa4f-74c818ccfe19 dockerRepository: airbyte/source-rki-covid - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/rki-covid sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9e43f34909b07..389c0929d5049 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7148,20 +7148,23 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-rki-covid:0.1.0" +- dockerImage: "airbyte/source-rki-covid:0.1.1" spec: - documentationUrl: "https://docsurl.com" + documentationUrl: "https://docs.airbyte.com/integrations/sources/rki-covid" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Rki Covid Spec" + title: "RKI Covid Spec" type: "object" - required: [] + required: + - "start_date" additionalProperties: false properties: start_date: type: "string" + title: "Start Date" description: "UTC date in the format 2017-01-25. Any data before this date\ \ will not be replicated." + order: 1 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: []