Separate Innovation, LTS and old releases of MySQL backend (#33522)

* Separate Innovation, LTS and old releases of MySQL backend

* Run tests against 'innovation' mysql releases

Author: Taragolis

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -730,11 +730,11 @@ def enter_shell(**kwargs) -> RunCommandResult:
         cmd.extend(["-c", cmd_added])
     if "arm64" in DOCKER_DEFAULT_PLATFORM:
         if shell_params.backend == "mysql":
-            if shell_params.mysql_version == "8":
+            if not shell_params.mysql_version.startswith("5"):
                 get_console().print("\n[warn]MySQL use MariaDB client binaries on ARM architecture.[/]\n")
             else:
                 get_console().print(
-                    f"\n[error]Only MySQL 8.0 is supported on ARM architecture, "
+                    f"\n[error]Only MySQL 8.x is supported on ARM architecture, "
                     f"but got {shell_params.mysql_version}[/]\n"
                 )
                 sys.exit(1)

dev/breeze/src/airflow_breeze/global_constants.py

@@ -84,7 +84,19 @@
 
 ALLOWED_MOUNT_OPTIONS = [MOUNT_SELECTED, MOUNT_ALL, MOUNT_SKIP, MOUNT_REMOVE]
 ALLOWED_POSTGRES_VERSIONS = ["11", "12", "13", "14", "15"]
-ALLOWED_MYSQL_VERSIONS = ["5.7", "8"]
+# Oracle introduced new release model for MySQL
+# - LTS: Long Time Support releases, new release approx every 2 year,
+#  with 5 year premier and 3 year extended support, no new features/removals during current LTS release.
+#  the first LTS release should be in summer/fall 2024.
+# - Innovations: Shot living releases with short support cycle - only until next Innovation/LTS release.
+# See: https://dev.mysql.com/blog-archive/introducing-mysql-innovation-and-long-term-support-lts-versions/
+MYSQL_LTS_RELEASES: list[str] = []
+MYSQL_OLD_RELEASES = ["5.7", "8.0"]
+MYSQL_INNOVATION_RELEASE = "8.1"
+ALLOWED_MYSQL_VERSIONS = [*MYSQL_OLD_RELEASES, *MYSQL_LTS_RELEASES]
+if MYSQL_INNOVATION_RELEASE:
+    ALLOWED_MYSQL_VERSIONS.append(MYSQL_INNOVATION_RELEASE)
+
 ALLOWED_MSSQL_VERSIONS = ["2017-latest", "2019-latest"]
 
 PIP_VERSION = "23.2.1"
@@ -190,7 +202,11 @@ def get_default_platform_machine() -> str:
 CURRENT_PYTHON_MAJOR_MINOR_VERSIONS = ALL_PYTHON_MAJOR_MINOR_VERSIONS
 CURRENT_POSTGRES_VERSIONS = ["11", "12", "13", "14", "15"]
 DEFAULT_POSTGRES_VERSION = CURRENT_POSTGRES_VERSIONS[0]
-CURRENT_MYSQL_VERSIONS = ["5.7", "8"]
+USE_MYSQL_INNOVATION_RELEASE = True
+if USE_MYSQL_INNOVATION_RELEASE:
+    CURRENT_MYSQL_VERSIONS = ALLOWED_MYSQL_VERSIONS.copy()
+else:
+    CURRENT_MYSQL_VERSIONS = [*MYSQL_OLD_RELEASES, *MYSQL_LTS_RELEASES]
 DEFAULT_MYSQL_VERSION = CURRENT_MYSQL_VERSIONS[0]
 CURRENT_MSSQL_VERSIONS = ["2017-latest", "2019-latest"]
 DEFAULT_MSSQL_VERSION = CURRENT_MSSQL_VERSIONS[0]

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -51,6 +51,7 @@
     CacheableChoice,
     CacheableDefault,
     DryRunOption,
+    MySQLBackendVersionType,
     UseAirflowVersionType,
     VerboseOption,
 )
@@ -151,7 +152,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     "-M",
     "--mysql-version",
     help="Version of MySQL used.",
-    type=CacheableChoice(ALLOWED_MYSQL_VERSIONS),
+    type=MySQLBackendVersionType(ALLOWED_MYSQL_VERSIONS),
     default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]),
     show_default=True,
 )

dev/breeze/src/airflow_breeze/utils/custom_param_types.py

@@ -191,6 +191,28 @@ def __init__(self, choices, case_sensitive: bool = True) -> None:
         super().__init__(choices=choices, case_sensitive=case_sensitive)
 
 
+class MySQLBackendVersionType(CacheableChoice):
+    def convert(self, value, param, ctx):
+        if isinstance(value, CacheableDefault):
+            param_name = param.envvar if param.envvar else param.name.upper()
+            mysql_version = read_from_cache_file(param_name)
+            if mysql_version == "8":
+                value = "8.0"
+                get_console().print(
+                    f"\n[warning]Found outdated cached value {mysql_version} for parameter {param.name}. "
+                    f"Replaced by {value}"
+                )
+                write_to_cache_file(param_name, "8.0", check_allowed_values=False)
+        else:
+            if value == "8":
+                value = "8.0"
+                get_console().print(
+                    f"\n[warning]Provided outdated value {8} for parameter {param.name}. "
+                    f"Will use {value} instead"
+                )
+        return super().convert(value, param, ctx)
+
+
 class UseAirflowVersionType(BetterChoice):
     """Extends choice with dynamic version number."""
 

images/breeze/output-commands-hash.txt

@@ -1,7 +1,7 @@
 # This file is automatically generated by pre-commit. If you have a conflict with this file
 # Please do not solve it but run `breeze setup regenerate-command-images`.
 # This command should fix the conflict and regenerate help images that you have conflict with.
-main:344261ca3aa7ff31e098b1d88280566a
+main:42493af4045e28dcf52040810fbb7036
 build-docs:bfc91db55c595516fe84d56c96a5a8e0
 ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee
 ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96
@@ -57,16 +57,16 @@ sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64
 sbom:935d041028e847d3faf763a95b51063e
 setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e
 setup:check-all-params-in-groups:639dfb061b8da02dcaa4a57b567ff42a
-setup:config:38ebaaf93ed42bc7b2a3000eeea2631d
+setup:config:fd32471ee31894decf91984615771add
 setup:regenerate-command-images:d5e29ec6acb1a6af7d83772c2962f89d
 setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b
 setup:version:be116d90a21c2afe01087f7609774e1e
-setup:8de3ed2645928e8f9f89c58f0ccd2a60
-shell:87e7bdcebe1180395adfec86b2a065f5
-start-airflow:f7216a8126ecf14b033e2ea677e1a105
+setup:fd391bab5277ad3aca65987a84144d51
+shell:c7adf57a354ecb496ae12fc0b9eaea80
+start-airflow:23df4528b3972977e58f7d29336500f7
 static-checks:5ded21248cd4f5617779f58ecf6c554c
 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507
 testing:helm-tests:8e491da2e01ebd815322c37562059d77
-testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3
-testing:tests:3c202e65824e405269e78f58936980e0
-testing:68a089bc30e0e60f834f205df1e9f086
+testing:integration-tests:f2a400ac9f722b9c279abbd4e3313e71
+testing:tests:abc232b1a9b416928ccc0a7dd904ed18
+testing:e6d79999174876ea42e73454f0ba9342