airbyte-ci: burst gradle task cache on new java cdk release (#36480)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 4.6.4   | [#36480](https://github.com/airbytehq/airbyte/pull/36480)  | Burst the Gradle Task cache if a new CDK version was released  |
 | 4.6.3   | [#36527](https://github.com/airbytehq/airbyte/pull/36527)  | Handle extras as well as groups in `airbyte ci test` [poetry packages]                                                     |
 | 4.6.2   | [#36220](https://github.com/airbytehq/airbyte/pull/36220)  | Allow using `migrate-to-base-image` without PULL_REQUEST_NUMBER                                                            |
 | 4.6.1   | [#36319](https://github.com/airbytehq/airbyte/pull/36319)  | Fix `ValueError` related to PR number in migrate-to-poetry                                                                 |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py

@@ -1,11 +1,13 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+import xml.etree.ElementTree as ET
 from abc import ABC
 from datetime import datetime
 from typing import Any, ClassVar, List, Optional, Tuple, cast
 
 import pipelines.dagger.actions.system.docker
+import requests
 from dagger import CacheSharingMode, CacheVolume, Container, ExecError
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.consts import AMAZONCORRETTO_IMAGE
@@ -32,6 +34,9 @@ class GradleTask(Step, ABC):
     GRADLE_DEP_CACHE_PATH = "/root/gradle-cache"
     GRADLE_HOME_PATH = "/root/.gradle"
     STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain")
+    CDK_MAVEN_METADATA_URL = (
+        "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/cdk/airbyte-cdk-core/maven-metadata.xml"
+    )
     gradle_task_name: ClassVar[str]
     bind_to_docker_host: ClassVar[bool] = False
     mount_connector_secrets: ClassVar[bool] = False
@@ -65,6 +70,14 @@ def _get_gradle_command(self, task: str, *args: Any, task_options: Optional[List
         task_options = task_options or []
         return f"./gradlew {' '.join(self.gradle_task_options + args)} {task} {' '.join(task_options)}"
 
+    def get_last_cdk_update_time(self) -> str:
+        response = requests.get(self.CDK_MAVEN_METADATA_URL)
+        response.raise_for_status()
+        last_updated = ET.fromstring(response.text).find(".//lastUpdated")
+        if last_updated is None or last_updated.text is None:
+            raise ValueError(f"Could not find the lastUpdated field in the CDK maven metadata at {self.CDK_MAVEN_METADATA_URL}")
+        return last_updated.text
+
     async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
         include = [
             ".root",
@@ -150,6 +163,8 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
             gradle_container_base
             # Mount the whole repo.
             .with_directory("/airbyte", self.context.get_repo_dir("."))
+            # Burst the cache if a new CDK version was released.
+            .with_env_variable("CDK_LAST_UPDATE", self.get_last_cdk_update_time())
             # Update the cache in place by executing a gradle task which will update all dependencies.
             .with_exec(
                 sh_dash_c(

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.6.3"
+version = "4.6.4"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]