Deprecate PY* constants into the airflow module (#37575)

Author: Taragolis

airflow/__init__.py

@@ -39,7 +39,7 @@
 # configuration is therefore initted early here, simply by importing it.
 from airflow import configuration, settings
 
-__all__ = ["__version__", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "PY311", "XComArg"]
+__all__ = ["__version__", "DAG", "Dataset", "XComArg"]
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user
@@ -55,13 +55,6 @@
 if not os.environ.get("_AIRFLOW__AS_LIBRARY", None):
     settings.initialize()
 
-PY36 = sys.version_info >= (3, 6)
-PY37 = sys.version_info >= (3, 7)
-PY38 = sys.version_info >= (3, 8)
-PY39 = sys.version_info >= (3, 9)
-PY310 = sys.version_info >= (3, 10)
-PY311 = sys.version_info >= (3, 11)
-
 # Things to lazy import in form {local_name: ('target_module', 'target_name', 'deprecated')}
 __lazy_imports: dict[str, tuple[str, str, bool]] = {
     "DAG": (".models.dag", "DAG", False),
@@ -77,6 +70,16 @@ def __getattr__(name: str):
     # PEP-562: Lazy loaded attributes on python modules
     module_path, attr_name, deprecated = __lazy_imports.get(name, ("", "", False))
     if not module_path:
+        if name.startswith("PY3") and (py_minor := name[3:]) in ("6", "7", "8", "9", "10", "11", "12"):
+            import warnings
+
+            warnings.warn(
+                f"Python version constraint {name!r} is deprecated and will be removed in the future. "
+                f"Please get version info from the 'sys.version_info'.",
+                DeprecationWarning,
+            )
+            return sys.version_info >= (3, int(py_minor))
+
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
     elif deprecated:
         import warnings

airflow/utils/hashlib_wrapper.py

@@ -18,13 +18,12 @@
 from __future__ import annotations
 
 import hashlib
+import sys
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from _typeshed import ReadableBuffer
 
-from airflow import PY39
-
 
 def md5(__string: ReadableBuffer = b"") -> hashlib._Hash:
     """
@@ -33,6 +32,6 @@ def md5(__string: ReadableBuffer = b"") -> hashlib._Hash:
     :param __string: The data to hash. Default to empty str byte.
     :return: The hashed value.
     """
-    if PY39:
+    if sys.version_info >= (3, 9):
         return hashlib.md5(__string, usedforsecurity=False)  # type: ignore
     return hashlib.md5(__string)

pyproject.toml

@@ -1462,8 +1462,17 @@ combine-as-imports = true
 banned-module-level-imports = ["numpy", "pandas"]
 
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
+# Direct import from the airflow package modules and constraints
 "airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException instead."
 "airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
+"airflow.PY36".msg = "Use sys.version_info >= (3, 6) instead."
+"airflow.PY37".msg = "Use sys.version_info >= (3, 7) instead."
+"airflow.PY38".msg = "Use sys.version_info >= (3, 8) instead."
+"airflow.PY39".msg = "Use sys.version_info >= (3, 9) instead."
+"airflow.PY310".msg = "Use sys.version_info >= (3, 10) instead."
+"airflow.PY311".msg = "Use sys.version_info >= (3, 11) instead."
+"airflow.PY312".msg = "Use sys.version_info >= (3, 12) instead."
+# Deprecated imports
 "airflow.models.baseoperator.BaseOperatorLink".msg = "Use airflow.models.baseoperatorlink.BaseOperatorLink"
 # Deprecated in Python 3.11, Pending Removal in Python 3.15: https://github.com/python/cpython/issues/90817
 # Deprecation warning in Python 3.11 also recommends using locale.getencoding but it available in Python 3.11

tests/core/test_airflow_module.py

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+
+import pytest
+
+import airflow
+from airflow.exceptions import AirflowException
+
+TEST_CASES = {
+    "PY36": sys.version_info >= (3, 6),
+    "PY37": sys.version_info >= (3, 7),
+    "PY38": sys.version_info >= (3, 8),
+    "PY39": sys.version_info >= (3, 9),
+    "PY310": sys.version_info >= (3, 10),
+    "PY311": sys.version_info >= (3, 11),
+    "PY312": sys.version_info >= (3, 12),
+}
+
+
+@pytest.mark.parametrize("py_attr, expected", TEST_CASES.items())
+def test_lazy_load_py_versions(py_attr, expected):
+    with pytest.warns(DeprecationWarning, match=f"Python version constraint '{py_attr}' is deprecated"):
+        # If there is no warning, then most possible it imported somewhere else.
+        assert getattr(airflow, py_attr) is expected
+
+
+@pytest.mark.parametrize("py_attr", ["PY35", "PY313"])
+def test_wrong_py_version(py_attr):
+    with pytest.raises(AttributeError, match=f"'airflow' has no attribute '{py_attr}'"):
+        getattr(airflow, py_attr)
+
+
+def test_deprecated_exception():
+    warning_pattern = "Import 'AirflowException' directly from the airflow module is deprecated"
+    with pytest.warns(DeprecationWarning, match=warning_pattern):
+        # If there is no warning, then most possible it imported somewhere else.
+        assert getattr(airflow, "AirflowException") is AirflowException

tests/dag_processing/test_processor.py

@@ -19,13 +19,14 @@
 
 import datetime
 import os
+import sys
 from unittest import mock
 from unittest.mock import MagicMock, patch
 from zipfile import ZipFile
 
 import pytest
 
-from airflow import PY311, settings
+from airflow import settings
 from airflow.callbacks.callback_requests import TaskCallbackRequest
 from airflow.configuration import TEST_DAGS_FOLDER, conf
 from airflow.dag_processing.manager import DagFileProcessorAgent
@@ -53,6 +54,7 @@
 pytestmark = pytest.mark.db_test
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+PY311 = sys.version_info >= (3, 11)
 
 # Include the words "airflow" and "dag" in the file contents,
 # tricking airflow into thinking these

tests/decorators/test_python.py

@@ -22,7 +22,6 @@
 
 import pytest
 
-from airflow import PY38, PY311
 from airflow.decorators import setup, task as task_decorator, teardown
 from airflow.decorators.base import DecoratedMappedOperator
 from airflow.exceptions import AirflowException, XComNotFound
@@ -49,6 +48,8 @@
     from airflow.models.dagrun import DagRun
 
 DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+PY38 = sys.version_info >= (3, 8)
+PY311 = sys.version_info >= (3, 11)
 
 
 class TestAirflowTaskDecorator(BasePythonTest):

tests/operators/test_python.py

@@ -37,7 +37,6 @@
 import pytest
 from slugify import slugify
 
-from airflow import PY311
 from airflow.decorators import task_group
 from airflow.exceptions import AirflowException, DeserializingResultError, RemovedInAirflow3Warning
 from airflow.models.baseoperator import BaseOperator
@@ -75,6 +74,7 @@
 TEMPLATE_SEARCHPATH = os.path.join(AIRFLOW_MAIN_FOLDER, "tests", "config_templates")
 LOGGER_NAME = "airflow.task.operators"
 DEFAULT_PYTHON_VERSION = f"{sys.version_info[0]}.{sys.version_info[1]}"
+PY311 = sys.version_info >= (3, 11)
 
 
 class BasePythonTest:

tests/providers/apache/hive/hooks/test_hive.py

@@ -17,22 +17,13 @@
 # under the License.
 from __future__ import annotations
 
-import pytest
-
-from airflow import PY311
-
-if PY311:
-    pytest.skip(
-        "The tests are skipped because Apache Hive provider is not supported on Python 3.11",
-        allow_module_level=True,
-    )
-
 import datetime
 import itertools
 from collections import namedtuple
 from unittest import mock
 
 import pandas as pd
+import pytest
 from hmsclient import HMSClient
 
 from airflow.exceptions import AirflowException

tests/serialization/serializers/test_serializers.py

@@ -18,6 +18,7 @@
 
 import datetime
 import decimal
+import sys
 from importlib import metadata
 from unittest.mock import patch
 
@@ -31,11 +32,10 @@
 from pendulum import DateTime
 from pendulum.tz.timezone import FixedTimezone, Timezone
 
-from airflow import PY39
 from airflow.models.param import Param, ParamsDict
 from airflow.serialization.serde import DATA, deserialize, serialize
 
-if PY39:
+if sys.version_info >= (3, 9):
     from zoneinfo import ZoneInfo
 else:
     from backports.zoneinfo import ZoneInfo