diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb0545eddffa6..7cf0454780bdd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -372,6 +372,13 @@ repos: require_serial: true files: ^airflow/providers/.*\.py$ additional_dependencies: ['rich>=12.4.4'] + - id: check-provider-yaml-value-importable + name: Check values in provider.yaml are importable + entry: ./scripts/ci/pre_commit/pre_commit_provider_import_path_exists.py + language: python + pass_filenames: true + files: ^airflow/providers/.*$ + additional_dependencies: ['pyyaml'] - id: update-local-yml-file name: Update mounts in the local yml file entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 9b5d1d8df7aaf..205e7eebabada 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -219,6 +219,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-provider-yaml-valid | Validate provider.yaml files | * | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-provider-yaml-value-importable | Check values in provider.yaml are importable | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-providers-init-file-missing | Provider init file is missing | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-providers-subpackages-init-file-exist | Provider subpackage init files are there | | diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index 51146a1e6aa69..3c10b9c05f818 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -59,5 +59,7 @@ hooks: - airflow.providers.apache.pinot.hooks.pinot connection-types: - - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotHook + - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook connection-type: pinot + - hook-class-name: airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook + connection-type: pinot_admin diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index cb2c85d3c855a..98797d4663f30 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -68,4 +68,4 @@ connection-types: connection-type: redis logging: - - airflow.providers.redis.redis_task_handler.RedisTaskHandler + - airflow.providers.redis.log.redis_task_handler.RedisTaskHandler diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index b6195dc04a8d4..9f46d284d40cb 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -82,4 +82,4 @@ connection-types: connection-type: slackwebhook notifications: - - airflow.providers.slack.notifications.slack_notifier.SlackNotifier + - airflow.providers.slack.notifications.slack.SlackNotifier diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 79e0cecba88c9..6dd7d693ad9e5 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -63,6 +63,7 @@ "check-pre-commit-information-consistent", "check-provide-create-sessions-imports", "check-provider-yaml-valid", + "check-provider-yaml-value-importable", "check-providers-init-file-missing", "check-providers-subpackages-init-file-exist", "check-pydevd-left-in-code", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 5c81b1f82538e..3017edbd264a2 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -64,7 +64,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:fd391bab5277ad3aca65987a84144d51 shell:c7adf57a354ecb496ae12fc0b9eaea80 start-airflow:23df4528b3972977e58f7d29336500f7 -static-checks:5ded21248cd4f5617779f58ecf6c554c +static-checks:3c6b570f3a05f12cc08eb0d0d2462c29 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:f2a400ac9f722b9c279abbd4e3313e71 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 51829ca34b0dc..8b9d38fa91877 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -217,59 +217,59 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8.0 | 8.1)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documents.                                                                                │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8.0 | 8.1)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documents.                                                                                │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index 9c029a855df92..eb0eec021eba3 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -1,4 +1,4 @@ - + - + @@ -270,9 +270,12 @@ + + + - Command: static-checks + Command: static-checks @@ -283,81 +286,82 @@ -Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... +Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]... Run static checks. -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--type-tType(s) of the static checks to run.                                             │ -│(all | black | blacken-docs | check-aiobotocore-optional |                       │ -│check-airflow-k8s-not-used | check-airflow-provider-compatibility |              │ -│check-apache-license-rat | check-base-operator-partial-arguments |               │ -│check-base-operator-usage | check-boring-cyborg-configuration |                  │ -│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ -│check-changelog-has-no-duplicates | check-cncf-k8s-only-for-executors |          │ -│check-core-deprecation-classes | check-daysago-import-from-utils |               │ -│check-decorated-operator-implements-custom-name | check-deferrable-default-value │ -│| check-docstring-param-types | check-example-dags-urls |                        │ -│check-executables-have-shebangs | check-extra-packages-references |              │ -│check-extras-order | check-for-inclusive-language |                              │ -│check-google-re2-as-dependency | check-hooks-apply |                             │ -│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ -│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ -│check-merge-conflict | check-newsfragments-are-valid |                           │ -│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ -│check-no-relative-imports | check-only-new-session-with-provide-session |        │ -│check-persist-credentials-disabled-in-github-workflows |                         │ -│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ -│check-provider-yaml-valid | check-providers-init-file-missing |                  │ -│check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |        │ -│check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order | │ -│check-start-date-not-used-in-defaults | check-system-tests-present |             │ -│check-system-tests-tocs | check-tests-unittest-testcase |                        │ -│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ -│| compile-www-assets | compile-www-assets-dev |                                  │ -│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ -│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ -│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ -│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ -│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ -│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ -│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ -│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ -│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ -│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-local-yml-file |                │ -│update-migration-references | update-providers-dependencies |                    │ -│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ -│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ -│--show-diff-on-failure-sShow diff for files modified by the checks.│ -│--initialize-environmentInitialize environment before running checks.│ -│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ -│(INTEGER RANGE)                                                       │ -│[default: 3; 1<=x<=10]                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ -│--file-fList of files to run the checks on.(PATH)│ -│--all-files-aRun checks on all files.│ -│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ -│exclusive with --last-commit.                                                                │ -│(TEXT)                                                                                       │ -│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ -│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ -│branch and HEAD of your branch.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ -│--skip-image-checkSkip checking if the CI image is up to date. Useful if you run non-image checks only│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--type-tType(s) of the static checks to run.                                             │ +│(all | black | blacken-docs | check-aiobotocore-optional |                       │ +│check-airflow-k8s-not-used | check-airflow-provider-compatibility |              │ +│check-apache-license-rat | check-base-operator-partial-arguments |               │ +│check-base-operator-usage | check-boring-cyborg-configuration |                  │ +│check-breeze-top-dependencies-limited | check-builtin-literals |                 │ +│check-changelog-has-no-duplicates | check-cncf-k8s-only-for-executors |          │ +│check-core-deprecation-classes | check-daysago-import-from-utils |               │ +│check-decorated-operator-implements-custom-name | check-deferrable-default-value │ +│| check-docstring-param-types | check-example-dags-urls |                        │ +│check-executables-have-shebangs | check-extra-packages-references |              │ +│check-extras-order | check-for-inclusive-language |                              │ +│check-google-re2-as-dependency | check-hooks-apply |                             │ +│check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |           │ +│check-lazy-logging | check-links-to-example-dags-do-not-use-hardcoded-versions | │ +│check-merge-conflict | check-newsfragments-are-valid |                           │ +│check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples |│ +│check-no-relative-imports | check-only-new-session-with-provide-session |        │ +│check-persist-credentials-disabled-in-github-workflows |                         │ +│check-pre-commit-information-consistent | check-provide-create-sessions-imports |│ +│check-provider-yaml-valid | check-provider-yaml-value-importable |               │ +│check-providers-init-file-missing | check-providers-subpackages-init-file-exist |│ +│check-pydevd-left-in-code | check-revision-heads-map |                           │ +│check-safe-filter-usage-in-html | check-setup-order |                            │ +│check-start-date-not-used-in-defaults | check-system-tests-present |             │ +│check-system-tests-tocs | check-tests-unittest-testcase |                        │ +│check-urlparse-usage-in-code | check-usage-of-re2-over-re | check-xml | codespell│ +│| compile-www-assets | compile-www-assets-dev |                                  │ +│create-missing-init-py-files-tests | debug-statements | detect-private-key |     │ +│doctoc | end-of-file-fixer | fix-encoding-pragma | flynt | identity |            │ +│insert-license | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart│ +│| lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core│ +│| mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | shellcheck |                   │ +│trailing-whitespace | ts-compile-format-lint-www | update-black-version |        │ +│update-breeze-cmd-output | update-breeze-readme-config-hash |                    │ +│update-common-sql-api-stubs | update-er-diagram | update-extras |                │ +│update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ +│update-installed-providers-to-be-sorted | update-local-yml-file |                │ +│update-migration-references | update-providers-dependencies |                    │ +│update-spelling-wordlist-to-be-sorted | update-supported-versions |              │ +│update-vendored-in-k8s-json-schema | update-version | yamllint)                  │ +│--show-diff-on-failure-sShow diff for files modified by the checks.│ +│--initialize-environmentInitialize environment before running checks.│ +│--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ +│(INTEGER RANGE)                                                       │ +│[default: 3; 1<=x<=10]                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +│--file-fList of files to run the checks on.(PATH)│ +│--all-files-aRun checks on all files.│ +│--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually    │ +│exclusive with --last-commit.                                                                │ +│(TEXT)                                                                                       │ +│--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref.│ +│--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main` │ +│branch and HEAD of your branch.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ +│--skip-image-checkSkip checking if the CI image is up to date. Useful if you run non-image checks only│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)[default: autodetect]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_provider_import_path_exists.py b/scripts/ci/pre_commit/pre_commit_provider_import_path_exists.py new file mode 100755 index 0000000000000..edcb9df69a5f6 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_provider_import_path_exists.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import ast +import functools +import itertools +import os +import pathlib +import sys +import typing + +import yaml + +if typing.TYPE_CHECKING: + import typing_extensions + +AIRFLOW_PROVIDERS_ROOT = pathlib.Path("airflow/providers") + +CheckType = typing.Union[ + None, # Checking for the module only. + typing.Type[ast.ClassDef], + typing.Type[ast.FunctionDef], +] + +# Tuple items are: +# - Section: Which top-level section in provider.yaml to check. +# - Subkey: If None, each value in the section is treated as an import path. +# If not None, each value is a dict, with the value to the specified key +# being an import path. +# - Check type: If None, the value is a module path. Otherwise the value is a +# (module-global) object's full name and this specifies its AST type. +SECTIONS_TO_CHECK: list[tuple[str, str | None, CheckType]] = [ + ("auth-backends", None, None), + ("connection-types", "hook-class-name", ast.ClassDef), + ("executors", None, ast.ClassDef), + ("extra-links", None, ast.ClassDef), + ("secrets-backends", None, ast.ClassDef), + ("logging", None, ast.ClassDef), + ("notifications", None, ast.ClassDef), + ("task-decorators", "class-name", ast.FunctionDef), +] + + +@functools.lru_cache +def _get_module(module_name: str) -> ast.Module | None: + path = pathlib.Path(*module_name.split(".")).with_suffix(".py") + if not path.is_file(): + return None + return ast.parse(path.read_text("utf-8"), os.fspath(path)) + + +def _is_check_type(v: ast.AST, t: type) -> typing_extensions.TypeGuard[ast.ClassDef | ast.FunctionDef]: + return isinstance(v, t) + + +def _iter_unimportable_paths( + data: dict, + section: str, + subkey: str | None, + check_type: CheckType, +) -> typing.Iterator[str]: + for item in data.get(section, ()): + if subkey is None: + import_path = item + else: + import_path = item[subkey] + + if check_type is None: + module_name = import_path + class_name = "" + else: + module_name, class_name = import_path.rsplit(".", 1) + + if (module := _get_module(module_name)) is None: + yield import_path + continue + + if check_type is None: + continue + + has_class = any( + _is_check_type(child, check_type) and child.name == class_name + for child in ast.iter_child_nodes(module) + ) + if not has_class: + yield import_path + + +def _iter_provider_yaml_errors(path: pathlib.Path) -> typing.Iterator[tuple[pathlib.Path, str, str]]: + with path.open() as f: + data = yaml.safe_load(f) + for section, subkey, check_type in SECTIONS_TO_CHECK: + for error in _iter_unimportable_paths(data, section, subkey, check_type): + yield path, section, error + + +def _iter_provider_paths(argv: list[str]) -> typing.Iterator[pathlib.Path]: + for argument in argv: + parents = list(pathlib.Path(argument).parents) + try: + root_index = parents.index(AIRFLOW_PROVIDERS_ROOT) + except ValueError: # Not in providers directory. + continue + for parent in parents[:root_index]: + if (provider_yaml := parent.joinpath("provider.yaml")).is_file(): + yield provider_yaml + + +def main(argv: list[str]) -> int: + providers_to_check = sorted(set(_iter_provider_paths(argv))) + errors = list(itertools.chain.from_iterable(_iter_provider_yaml_errors(p) for p in providers_to_check)) + for provider, section, value in errors: + print(f"Broken {provider.relative_to(AIRFLOW_PROVIDERS_ROOT)} {section!r}: {value}") + return len(errors) + + +if __name__ == "__main__": + sys.exit(main(sys.argv))