chore: migrate config change functions to common module (#3311)

In this PR:
- Migrate config change functions to `common` module.
- Library generation will accept library names directly.

Author: JoeWang1127

.github/scripts/action.yaml

@@ -60,8 +60,6 @@ runs:
       cd "${GITHUB_WORKSPACE}"
       pip install --require-hashes -r hermetic_build/common/requirements.txt
       pip install hermetic_build/common
-      pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
-      pip install hermetic_build/library_generation
       pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
       pip install hermetic_build/release_note_generation
   - name: Generate changed libraries

.github/scripts/hermetic_library_generation.sh

@@ -92,6 +92,12 @@ pushd "${api_def_dir}"
 git checkout "${googleapis_commitish}"
 popd
 
+# get changed library list.
+changed_libraries=$(python hermetic_build/common/cli/get_changed_libraries.py create \
+  --baseline-generation-config-path="${baseline_generation_config}" \
+  --current-generation-config-path="${generation_config}")
+echo "Changed libraries are: ${changed_libraries:-"No changed library"}."
+
 # run hermetic code generation docker image.
 docker run \
   --rm \
@@ -101,8 +107,8 @@ docker run \
   -v "${api_def_dir}:${workspace_name}/googleapis" \
   -e GENERATOR_VERSION="${image_tag}" \
   gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
-  --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
-  --current-generation-config-path="${workspace_name}/${generation_config}" \
+  --generation-config-path="${workspace_name}/${generation_config}" \
+  --library-names="${changed_libraries}" \
   --api-definitions-path="${workspace_name}/googleapis"
 
 python hermetic_build/release_note_generation/cli/generate_release_note.py generate \

hermetic_build/common/cli/get_changed_libraries.py

@@ -0,0 +1,83 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+import click as click
+
+from common.model.generation_config import from_yaml
+from common.utils.generation_config_comparator import compare_config
+
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+    pass
+
+
+@main.command()
+@click.option(
+    "--baseline-generation-config-path",
+    required=True,
+    type=str,
+    help="""
+    Absolute or relative path to a generation_config.yaml.
+    This config file is used for computing changed library list.
+    """,
+)
+@click.option(
+    "--current-generation-config-path",
+    required=True,
+    type=str,
+    help="""
+    Absolute or relative path to a generation_config.yaml that contains the
+    metadata about library generation.
+    """,
+)
+def create(
+    baseline_generation_config_path: str,
+    current_generation_config_path: str,
+) -> None:
+    """
+    Compares baseline generation config with current generation config and
+    generates changed library names (a comma separated string) based on current
+    generation config.
+    """
+    baseline_generation_config_path = os.path.abspath(baseline_generation_config_path)
+    if not os.path.isfile(baseline_generation_config_path):
+        raise FileNotFoundError(
+            f"{baseline_generation_config_path} does not exist. "
+            "A valid generation config has to be passed in as "
+            "baseline-generation-config-path."
+        )
+    current_generation_config_path = os.path.abspath(current_generation_config_path)
+    if not os.path.isfile(current_generation_config_path):
+        raise FileNotFoundError(
+            f"{current_generation_config_path} does not exist. "
+            "A valid generation config has to be passed in as "
+            "current-generation-config-path."
+        )
+    config_change = compare_config(
+        baseline_config=from_yaml(baseline_generation_config_path),
+        current_config=from_yaml(current_generation_config_path),
+    )
+    changed_libraries = config_change.get_changed_libraries()
+    if changed_libraries is None:
+        print("No changed library.")
+        return
+    click.echo(",".join(config_change.get_changed_libraries()))
+
+
+if __name__ == "__main__":
+    main()

hermetic_build/library_generation/model/config_change.py renamed to hermetic_build/common/model/config_change.py

@@ -11,17 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import shutil
+import tempfile
 from enum import Enum
 from typing import Optional
 from git import Commit, Repo
-
 from common.model.gapic_inputs import parse_build_str
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig
-from library_generation.utils.utilities import sh_util
-from library_generation.utils.proto_path_utils import find_versioned_proto_path
+from common.utils.proto_path_utils import find_versioned_proto_path
 
 INSERTIONS = "insertions"
 LINES = "lines"
@@ -109,25 +106,22 @@ def get_qualified_commits(
         :param repo_url: the repository contains the commit history.
         :return: QualifiedCommit objects.
         """
-        tmp_dir = sh_util("get_output_folder")
-        shutil.rmtree(tmp_dir, ignore_errors=True)
-        os.mkdir(tmp_dir)
-        # we only need commit history, thus shadow clone is enough.
-        repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"])
-        commit = repo.commit(self.current_config.googleapis_commitish)
-        proto_paths = self.current_config.get_proto_path_to_library_name()
-        qualified_commits = []
-        while str(commit.hexsha) != self.baseline_config.googleapis_commitish:
-            qualified_commit = ConfigChange.__create_qualified_commit(
-                proto_paths=proto_paths, commit=commit
-            )
-            if qualified_commit is not None:
-                qualified_commits.append(qualified_commit)
-            commit_parents = commit.parents
-            if len(commit_parents) == 0:
-                break
-            commit = commit_parents[0]
-        shutil.rmtree(tmp_dir, ignore_errors=True)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # we only need commit history, thus a shadow clone is enough.
+            repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"])
+            commit = repo.commit(self.current_config.googleapis_commitish)
+            proto_paths = self.current_config.get_proto_path_to_library_name()
+            qualified_commits = []
+            while str(commit.hexsha) != self.baseline_config.googleapis_commitish:
+                qualified_commit = ConfigChange.__create_qualified_commit(
+                    proto_paths=proto_paths, commit=commit
+                )
+                if qualified_commit is not None:
+                    qualified_commits.append(qualified_commit)
+                commit_parents = commit.parents
+                if len(commit_parents) == 0:
+                    break
+                commit = commit_parents[0]
         return qualified_commits
 
     def __get_library_names_from_qualified_commits(self) -> list[str]:

hermetic_build/common/requirements.in

@@ -1,3 +1,4 @@
 black==24.8.0
+GitPython==3.1.43
 parameterized==0.9.0
 PyYAML==6.0.2

hermetic_build/common/requirements.txt

@@ -32,6 +32,14 @@ click==8.1.7 \
     --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
     --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
     # via black
+gitdb==4.0.11 \
+    --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \
+    --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b
+    # via gitpython
+gitpython==3.1.43 \
+    --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \
+    --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff
+    # via -r hermetic_build/common/requirements.in
 mypy-extensions==1.0.0 \
     --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
     --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
@@ -107,3 +115,7 @@ pyyaml==6.0.2 \
     --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
     --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
     # via -r hermetic_build/common/requirements.in
+smmap==5.0.1 \
+    --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \
+    --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da
+    # via gitdb

hermetic_build/common/tests/cli/__init__.py

@@ -0,0 +1,71 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from click.testing import CliRunner
+import unittest
+
+from common.cli.get_changed_libraries import create
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+test_resource_dir = os.path.join(script_dir, "..", "resources", "cli")
+
+
+class GetChangedLibrariesTest(unittest.TestCase):
+    def test_entry_point_without_baseline_config_raise_system_exception(self):
+        os.chdir(script_dir)
+        runner = CliRunner()
+        # noinspection PyTypeChecker
+        result = runner.invoke(create)
+        self.assertEqual(2, result.exit_code)
+        self.assertEqual(SystemExit, result.exc_info[0])
+
+    def test_entry_point_without_current_config_raise_system_exception(self):
+        os.chdir(script_dir)
+        runner = CliRunner()
+        # noinspection PyTypeChecker
+        result = runner.invoke(
+            create, ["--baseline-generation-config-path=/invalid/path/file"]
+        )
+        self.assertEqual(2, result.exit_code)
+        self.assertEqual(SystemExit, result.exc_info[0])
+
+    def test_entry_point_with_invalid_baseline_config_raise_file_exception(self):
+        os.chdir(script_dir)
+        runner = CliRunner()
+        # noinspection PyTypeChecker
+        result = runner.invoke(
+            create,
+            [
+                "--baseline-generation-config-path=/invalid/path/file",
+                "--current-generation-config-path=/invalid/path/file",
+            ],
+        )
+        self.assertEqual(1, result.exit_code)
+        self.assertEqual(FileNotFoundError, result.exc_info[0])
+        self.assertRegex(result.exception.args[0], "baseline-generation-config-path")
+
+    def test_entry_point_with_invalid_current_config_raise_file_exception(self):
+        os.chdir(script_dir)
+        runner = CliRunner()
+        # noinspection PyTypeChecker
+        result = runner.invoke(
+            create,
+            [
+                f"--baseline-generation-config-path={test_resource_dir}/empty_config.yaml",
+                "--current-generation-config-path=/invalid/path/file",
+            ],
+        )
+        self.assertEqual(1, result.exit_code)
+        self.assertEqual(FileNotFoundError, result.exc_info[0])
+        self.assertRegex(result.exception.args[0], "current-generation-config-path")

hermetic_build/common/tests/cli/config_change_unit_tests.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 import unittest
 
-from library_generation.model.config_change import ChangeType
-from library_generation.model.config_change import ConfigChange
-from library_generation.model.config_change import LibraryChange
+from common.model.config_change import ChangeType
+from common.model.config_change import ConfigChange
+from common.model.config_change import LibraryChange
 from common.model.gapic_config import GapicConfig
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig

hermetic_build/library_generation/tests/model/config_change_unit_tests.py renamed to hermetic_build/common/tests/model/config_change_unit_tests.py

@@ -16,8 +16,8 @@
 from common.model.gapic_config import GapicConfig
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig
-from library_generation.utils.generation_config_comparator import ChangeType
-from library_generation.utils.generation_config_comparator import compare_config
+from common.utils.generation_config_comparator import ChangeType
+from common.utils.generation_config_comparator import compare_config
 
 
 class GenerationConfigComparatorTest(unittest.TestCase):

hermetic_build/common/tests/resources/cli/empty_config.yaml

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+#  Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+from pathlib import Path
+from common.utils.proto_path_utils import find_versioned_proto_path
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+resources_dir = os.path.join(script_dir, "..", "resources")
+test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve()
+
+
+class ProtoPathsUtilsTest(unittest.TestCase):
+    def test_find_versioned_proto_path_nested_version_success(self):
+        proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto"
+        expected = "google/cloud/aiplatform/v1"
+        self.assertEqual(expected, find_versioned_proto_path(proto_path))
+
+    def test_find_versioned_proto_path_success(self):
+        proto_path = "google/cloud/asset/v1p2beta1/assets.proto"
+        expected = "google/cloud/asset/v1p2beta1"
+        self.assertEqual(expected, find_versioned_proto_path(proto_path))
+
+    def test_find_versioned_proto_without_version_return_itself(self):
+        proto_path = "google/type/color.proto"
+        expected = "google/type/color.proto"
+        self.assertEqual(expected, find_versioned_proto_path(proto_path))

hermetic_build/common/tests/utils/__init__.py

@@ -15,10 +15,10 @@
 from typing import Any
 from typing import Dict
 from typing import List
-from library_generation.model.config_change import ChangeType
-from library_generation.model.config_change import ConfigChange
-from library_generation.model.config_change import LibraryChange
-from library_generation.model.config_change import HashLibrary
+from common.model.config_change import ChangeType
+from common.model.config_change import ConfigChange
+from common.model.config_change import LibraryChange
+from common.model.config_change import HashLibrary
 from common.model.gapic_config import GapicConfig
 from common.model.generation_config import GenerationConfig
 from common.model.library_config import LibraryConfig

hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py renamed to hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+#  Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+
+def find_versioned_proto_path(proto_path: str) -> str:
+    """
+    Returns a versioned proto_path from a given proto_path; or proto_path itself
+    if it doesn't contain a versioned proto_path.
+    :param proto_path: a proto file path
+    :return: the versioned proto_path
+    """
+    version_regex = re.compile(r"^v[1-9].*")
+    directories = proto_path.split("/")
+    for directory in directories:
+        result = version_regex.search(directory)
+        if result:
+            version = result[0]
+            idx = proto_path.find(version)
+            return proto_path[:idx] + version
+    return proto_path