feat: get released version from versions.txt to render README.md (#3007)

In this PR:
- get library version from `versions.txt` to render `README.md`.
- set library version as an env variable to post processor.
- add `distribution_name` checker in `LibraryConfig`.

Author: JoeWang1127

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -17,7 +17,7 @@ FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
-ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940
+ARG SYNTHTOOL_COMMITTISH=696c4bff721f5541cd75fdc97d413f8f39d2a2c1
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.3
 ENV HOME=/home

library_generation/generate_composed_library.py

@@ -36,6 +36,7 @@
 from library_generation.model.gapic_inputs import GapicInputs
 from library_generation.model.library_config import LibraryConfig
 from library_generation.model.gapic_inputs import parse as parse_build_file
+from library_generation.model.repo_config import RepoConfig
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 
@@ -44,8 +45,7 @@ def generate_composed_library(
     config: GenerationConfig,
     library_path: str,
     library: LibraryConfig,
-    output_folder: str,
-    versions_file: str,
+    repo_config: RepoConfig,
 ) -> None:
     """
     Generate libraries composed of more than one service or service version
@@ -55,10 +55,10 @@ def generate_composed_library(
     :param library_path: the path to which the generated file goes
     :param library: a LibraryConfig object contained inside config, passed here
     for convenience and to prevent all libraries to be processed
-    :param output_folder: the folder to where tools go
-    :param versions_file: the file containing version of libraries
+    :param repo_config:
     :return None
     """
+    output_folder = repo_config.output_folder
     util.pull_api_definition(
         config=config, library=library, output_folder=output_folder
     )
@@ -102,16 +102,20 @@ def generate_composed_library(
             cwd=output_folder,
         )
 
+    library_version = repo_config.get_library_version(
+        artifact_id=library.get_artifact_id()
+    )
     # call postprocess library
     util.run_process_and_print_output(
         [
             f"{script_dir}/postprocess_library.sh",
             f"{library_path}",
             "",
-            versions_file,
+            repo_config.versions_file,
             owlbot_cli_source_folder,
             str(config.is_monorepo()).lower(),
             config.libraries_bom_version,
+            library_version,
         ],
         "Library postprocessing",
     )

library_generation/generate_repo.py

@@ -44,14 +44,13 @@ def generate_from_yaml(
         gen_config=config, library_config=target_libraries, repo_path=repository_path
     )
 
-    for library_path, library in repo_config.libraries.items():
+    for library_path, library in repo_config.get_libraries().items():
         print(f"generating library {library.get_library_name()}")
         generate_composed_library(
             config=config,
             library_path=library_path,
             library=library,
-            output_folder=repo_config.output_folder,
-            versions_file=repo_config.versions_file,
+            repo_config=repo_config,
         )
 
     if not config.is_monorepo() or config.contains_common_protos():

library_generation/model/library_config.py

@@ -13,11 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from hashlib import sha256
-
 from typing import Optional
 from library_generation.model.gapic_config import GapicConfig
 
 
+MAVEN_COORDINATE_SEPARATOR = ":"
+
+
 class LibraryConfig:
     """
     Class that represents a library in a generation_config.yaml file
@@ -64,7 +66,6 @@ def __init__(
         self.excluded_dependencies = excluded_dependencies
         self.excluded_poms = excluded_poms
         self.client_documentation = client_documentation
-        self.distribution_name = distribution_name
         self.googleapis_commitish = googleapis_commitish
         self.group_id = group_id
         self.issue_tracker = issue_tracker
@@ -76,6 +77,7 @@ def __init__(
         self.extra_versioned_modules = extra_versioned_modules
         self.recommended_package = recommended_package
         self.min_java_version = min_java_version
+        self.distribution_name = self.__get_distribution_name(distribution_name)
 
     def get_library_name(self) -> str:
         """
@@ -87,6 +89,34 @@ def get_library_name(self) -> str:
     def get_sorted_gapic_configs(self) -> list[GapicConfig]:
         return sorted(self.gapic_configs)
 
+    def get_maven_coordinate(self) -> str:
+        """
+        Returns the Maven coordinate (group_id:artifact_id) of the library
+        """
+        return self.distribution_name
+
+    def get_artifact_id(self) -> str:
+        """
+        Returns the artifact ID of the library
+        """
+        return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1]
+
+    def __get_distribution_name(self, distribution_name: Optional[str]) -> str:
+        LibraryConfig.__check_distribution_name(distribution_name)
+        if distribution_name:
+            return distribution_name
+        cloud_prefix = "cloud-" if self.cloud_api else ""
+        library_name = self.get_library_name()
+        return f"{self.group_id}:google-{cloud_prefix}{library_name}"
+
+    @staticmethod
+    def __check_distribution_name(distribution_name: str) -> None:
+        if not distribution_name:
+            return
+        sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR)
+        if len(sections) != 2:
+            raise ValueError(f"{distribution_name} is not a valid distribution name.")
+
     def __eq__(self, other):
         return (
             self.api_shortname == other.api_shortname

library_generation/model/repo_config.py

@@ -12,10 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Dict
 from library_generation.model.library_config import LibraryConfig
 
 
+GRPC_PREFIX = "grpc-"
+PROTO_PREFIX = "proto-"
+NEW_CLIENT_VERSION = "0.0.0"
+
+
 class RepoConfig:
     """
     Class that represents a generated repository
@@ -24,15 +28,47 @@ class RepoConfig:
     def __init__(
         self,
         output_folder: str,
-        libraries: Dict[str, LibraryConfig],
+        libraries: dict[str, LibraryConfig],
         versions_file: str,
     ):
         """
         Init a RepoConfig object
+
         :param output_folder: the path to which the generated repo goes
         :param libraries: a mapping from library_path to LibraryConfig object
         :param versions_file: the path of versions.txt used in post-processing
         """
         self.output_folder = output_folder
         self.libraries = libraries
         self.versions_file = versions_file
+        self.library_versions = self.__parse_versions()
+
+    def get_libraries(self) -> dict[str, LibraryConfig]:
+        return self.libraries
+
+    def get_library_version(self, artifact_id: str) -> str:
+        """
+        Returns the version of a given artifact ID.
+        If the artifact ID is not managed, i.e., a new client, returns `0.0.0`.
+
+        :param artifact_id: the Maven artifact ID.
+        :return: the version of the artifact.
+        """
+        return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION)
+
+    def __parse_versions(self) -> dict[str, str]:
+        library_versions = dict()
+        with open(self.versions_file) as f:
+            for line in f.readlines():
+                sections = line.split(":")
+                # skip comments and empty lines.
+                if len(sections) != 3:
+                    continue
+                artifact_id = sections[0]
+                released_version = sections[1]
+                if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith(
+                    PROTO_PREFIX
+                ):
+                    continue
+                library_versions[artifact_id] = released_version
+        return library_versions

library_generation/owlbot/bin/entrypoint.sh

@@ -22,6 +22,7 @@
 # both to README and pom.xml files
 # 3: is_monorepo: whether we are postprocessing a monorepo
 # 4: libraries_bom_version: used to render the readme
+# 5: library_version: used to render the readme
 
 # The scripts assumes the CWD is the folder where postprocessing is going to be
 # applied
@@ -31,7 +32,7 @@ scripts_root=$1
 versions_file=$2
 is_monorepo=$3
 libraries_bom_version=$4
-
+library_version=$5
 
 if [[ "${is_monorepo}" == "true" ]]; then
   mv owl-bot-staging/* temp
@@ -48,10 +49,12 @@ then
   # synthtool library considering the way owlbot.py files are written
   export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates"
   export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}"
+  export SYNTHTOOL_LIBRARY_VERSION="${library_version}"
   # defaults to run owlbot.py
   python3 owlbot.py
   unset SYNTHTOOL_TEMPLATES
   unset SYNTHTOOL_LIBRARIES_BOM_VERSION
+  unset SYNTHTOOL_LIBRARY_VERSION
 fi
 echo "...done"
 

library_generation/owlbot/templates/java_library/README.md

@@ -1,5 +1,6 @@
 {% set group_id = metadata['repo']['distribution_name'].split(':')|first -%}
 {% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%}
+{% set version = metadata['library_version'] -%}
 {% set repo_short = metadata['repo']['repo'].split('/')|last -%}
 
 # Google {{ metadata['repo']['name_pretty'] }} Client for Java
@@ -71,7 +72,7 @@ If you are using Maven, add this to your pom.xml file:
 <dependency>
   <groupId>{{ group_id }}</groupId>
   <artifactId>{{ artifact_id }}</artifactId>
-  <version>{{ metadata['latest_version'] }}</version>
+  <version>{{ version }}</version>
 </dependency>
 {% endif -%}
 ```
@@ -80,7 +81,7 @@ If you are using Maven, add this to your pom.xml file:
 If you are using Gradle 5.x or later, add this to your dependencies:
 
 ```Groovy
-implementation platform('com.google.cloud:libraries-bom:{{metadata['latest_bom_version']}}')
+implementation platform('com.google.cloud:libraries-bom:{{metadata['libraries_bom_version']}}')
 
 implementation '{{ group_id }}:{{ artifact_id }}'
 ```
@@ -89,13 +90,13 @@ implementation '{{ group_id }}:{{ artifact_id }}'
 If you are using Gradle without BOM, add this to your dependencies:
 
 ```Groovy
-implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}'
+implementation '{{ group_id }}:{{ artifact_id }}:{{ version }}'
 ```
 
 If you are using SBT, add this to your dependencies:
 
 ```Scala
-libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}"
+libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ version }}"
 ```
 <!-- {x-version-update-end} -->
 
@@ -264,7 +265,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
 [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html
 [stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %}
 [maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg
-[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }}
+[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ version }}
 [authentication]: https://github.com/googleapis/google-cloud-java#authentication
 [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
 [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles

library_generation/postprocess_library.sh

@@ -19,6 +19,8 @@
 # different logic
 # 6 - libraries_bom_version: used by our implementation of owlbot to render the
 # readme
+# 7 - library_version: used by our implementation of owlbot to render the
+# readme
 set -exo pipefail
 scripts_root=$(dirname "$(readlink -f "$0")")
 
@@ -28,6 +30,7 @@ versions_file=$3
 owlbot_cli_source_folder=$4
 is_monorepo=$5
 libraries_bom_version=$6
+library_version=$7
 owlbot_yaml_file_name=".OwlBot-hermetic.yaml"
 
 source "${scripts_root}"/utils/utilities.sh
@@ -102,6 +105,7 @@ bash "${scripts_root}/owlbot/bin/entrypoint.sh" \
   "${scripts_root}" \
   "${versions_file}" \
   "${is_monorepo}" \
-  "${libraries_bom_version}"
+  "${libraries_bom_version}" \
+  "${library_version}"
 
 popd # postprocessing_target

library_generation/templates/owlbot.yaml.monorepo.j2

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{% if artifact_name %}
+{% if artifact_id %}
 deep-remove-regex:
 - "/{{ module_name }}/grpc-google-.*/src"
 - "/{{ module_name }}/proto-google-.*/src"
@@ -24,11 +24,11 @@ deep-preserve-regex:
 
 deep-copy-regex:
 - source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src"
-  dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_name }}-$1/src"
+  dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src"
 - source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src"
-  dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_name }}-$1/src"
+  dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src"
 - source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src"
-  dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_name }}/src"
+  dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src"
 - source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated"
   dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated"
 {%- endif %}

library_generation/test/model/library_config_unit_tests.py

@@ -64,3 +64,59 @@ def test_get_sorted_gapic_configs_returns_correct_order(self):
             ],
             library.get_sorted_gapic_configs(),
         )
+
+    def test_init_invalid_distribution_name_raise_value_error(self):
+        self.assertRaisesRegex(
+            ValueError,
+            "fake-distribution-name is not a valid distribution name.",
+            LibraryConfig,
+            api_shortname="baremetalsolution",
+            name_pretty="Bare Metal Solution",
+            product_documentation="https://cloud.google.com/bare-metal/docs",
+            api_description="example api description",
+            gapic_configs=list(),
+            distribution_name="fake-distribution-name",
+        )
+
+    def test_get_distribution_name_cloud_api(self):
+        library = LibraryConfig(
+            api_shortname="baremetalsolution",
+            name_pretty="Bare Metal Solution",
+            product_documentation="https://cloud.google.com/bare-metal/docs",
+            api_description="example api description",
+            gapic_configs=list(),
+        )
+        self.assertEqual(
+            "com.google.cloud:google-cloud-baremetalsolution",
+            library.get_maven_coordinate(),
+        )
+        self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id())
+
+    def test_get_distribution_name_non_cloud_api(self):
+        library = LibraryConfig(
+            api_shortname="baremetalsolution",
+            name_pretty="Bare Metal Solution",
+            product_documentation="https://cloud.google.com/bare-metal/docs",
+            api_description="example api description",
+            gapic_configs=list(),
+            cloud_api=False,
+            group_id="com.example",
+        )
+        self.assertEqual(
+            "com.example:google-baremetalsolution", library.get_maven_coordinate()
+        )
+        self.assertEqual("google-baremetalsolution", library.get_artifact_id())
+
+    def test_get_distribution_name_with_distribution_name(self):
+        library = LibraryConfig(
+            api_shortname="baremetalsolution",
+            name_pretty="Bare Metal Solution",
+            product_documentation="https://cloud.google.com/bare-metal/docs",
+            api_description="example api description",
+            gapic_configs=list(),
+            distribution_name="com.example:baremetalsolution",
+        )
+        self.assertEqual(
+            "com.example:baremetalsolution", library.get_maven_coordinate()
+        )
+        self.assertEqual("baremetalsolution", library.get_artifact_id())