feat: Move release note generation to a sub module (#3299)

In this PR:
- Separate projects to three modules: common, library_generation and
release_note_generation
- Add `generate_release_note.py` to separate PR description generation
from `entrypoint.py`.
- Only install common and library_generation module in image.
- Remove PR description comparison in integration test.

Example generation workflow in downstream library:
https://github.com/googleapis/java-bigtable/actions/runs/11442581326/job/31833887512

Note that `release_note_generation` module still depends on
`library_generation` module because we didn't separate config change
functions into `common` module. This will be in a follow up PR.

Author: JoeWang1127

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -45,19 +45,20 @@ RUN apt-get update && apt-get install -y \
 	&& apt-get clean
 
 # copy source code
-COPY library_generation /src
+COPY hermetic_build/common /src/common
+COPY hermetic_build/library_generation /src/library_generation
 
 # install protoc
 WORKDIR /protoc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
 ENV DOCKER_PROTOC_LOCATION=/protoc
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
@@ -71,16 +72,18 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
 RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
 
-#  use python 3.11 (the base image has several python versions; here we define the default one)
+#  use python 3.12 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
-RUN ln -s $(which python3.11) /usr/local/bin/python
-RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN ln -s $(which python3.12) /usr/local/bin/python
+RUN ln -s $(which python3.12) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
 
 # install main scripts as a python package
-WORKDIR /src
-RUN python -m pip install -r requirements.txt
-RUN python -m pip install .
+WORKDIR /
+RUN python -m pip install --require-hashes -r src/common/requirements.txt
+RUN python -m pip install src/common
+RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt
+RUN python -m pip install src/library_generation
 
 # Install nvm with node and npm
 ENV NODE_VERSION 20.12.0
@@ -120,4 +123,4 @@ RUN chmod -R a+rw /home
 RUN chmod -R a+rx /home/.nvm
 
 WORKDIR /workspace
-ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]
+ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ]

.github/scripts/action.yaml

@@ -37,11 +37,33 @@ inputs:
 runs:
   using: "composite"
   steps:
+  - uses: actions/setup-python@v5
+    with:
+      python-version: 3.12
+  - name: Copy python script
+    shell: bash
+    run: |
+      set -x
+      # repository root
+      cd ${{ github.action_path }}/../../
+      rsync -rv \
+        --exclude=tests \
+        hermetic_build "${GITHUB_WORKSPACE}"
   - name: Copy shell script
     shell: bash
     run: |
       cd ${{ github.action_path }}
-      cp hermetic_library_generation.sh $GITHUB_WORKSPACE
+      cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}"
+  - name: Install python packages
+    shell: bash
+    run: |
+      cd "${GITHUB_WORKSPACE}"
+      pip install --require-hashes -r hermetic_build/common/requirements.txt
+      pip install hermetic_build/common
+      pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+      pip install hermetic_build/library_generation
+      pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+      pip install hermetic_build/release_note_generation
   - name: Generate changed libraries
     shell: bash
     run: |

.github/scripts/hermetic_library_generation.sh

@@ -106,12 +106,16 @@ docker run \
   --current-generation-config-path="${workspace_name}/${generation_config}" \
   --api-definitions-path="${workspace_name}/googleapis"
 
+python hermetic_build/release_note_generation/cli/generate_release_note.py generate \
+  --baseline-generation-config-path="${baseline_generation_config}" \
+  --current-generation-config-path="${generation_config}"
+
 # remove api definitions after generation
 rm -rf "${api_def_dir}"
 
 # commit the change to the pull request.
 rm -rdf output googleapis "${baseline_generation_config}"
-git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh'
+git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' ':!hermetic_build'
 changed_files=$(git diff --cached --name-only)
 if [[ "${changed_files}" != "" ]]; then
     echo "Commit changes..."

.github/workflows/verify_library_generation.yaml

@@ -27,7 +27,7 @@ jobs:
           git checkout -b "${head_ref}" fork/${head_ref}
           changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")"
         fi
-        if [[ ${changed_directories} =~ "library_generation/" ]]; then
+        if [[ ${changed_directories} =~ "hermetic_build/" ]]; then
           echo "should_run=true" >> $GITHUB_OUTPUT
         else
           echo "should_run=false" >> $GITHUB_OUTPUT
@@ -46,32 +46,21 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
-    - name: install pyenv
+        python-version: 3.12
+    - name: install python modules and dependencies
       shell: bash
       run: |
         set -ex
-        curl https://pyenv.run | bash
-        # setup environment
-        export PYENV_ROOT="$HOME/.pyenv"
-        export PATH="$PYENV_ROOT/bin:$PATH"
-        echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
-        echo "PATH=${PATH}" >> $GITHUB_ENV
-
-        set +ex
-    - name: install python dependencies
-      shell: bash
-      run: |
-        set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
     - name: Run integration tests
       shell: bash
       run: |
         set -x
-        python -m unittest library_generation/test/integration_tests.py
+        python -m unittest hermetic_build/library_generation/tests/integration_tests.py
   library-generation-unit-tests:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -80,23 +69,26 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
-    - name: install python dependencies
+        python-version: 3.12
+    - name: install python modules and dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
+        pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+        pip install hermetic_build/release_note_generation
     - name: Run shell unit tests
       run: |
         set -x
-        library_generation/test/generate_library_unit_tests.sh
+        hermetic_build/library_generation/tests/generate_library_unit_tests.sh
     - name: Run python unit tests
       run: |
         set -x
-        python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
+        python -m unittest discover -s hermetic_build -p "*unit_tests.py"
   library-generation-lint-shell:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -106,7 +98,7 @@ jobs:
     - name: Run ShellCheck
       uses: ludeeus/[email protected]
       with:
-        scandir: 'library_generation'
+        scandir: 'hermetic_build'
         format: tty
         severity: error
   library-generation-lint-python:
@@ -115,16 +107,23 @@ jobs:
     if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
     - name: install python dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
+        pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+        pip install hermetic_build/release_note_generation
     - name: Lint
       shell: bash
       run: |
         # exclude generated golden files
         # exclude owlbot until further refaction
-        black --check library_generation --exclude "(library_generation/test/resources/goldens)"
+        black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)"

library_generation/__init__.py hermetic_build/common/__init__.py

@@ -16,8 +16,8 @@
 
 import yaml
 from typing import Optional
-from library_generation.model.library_config import LibraryConfig
-from library_generation.model.gapic_config import GapicConfig
+from common.model.library_config import LibraryConfig
+from common.model.gapic_config import GapicConfig
 
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"

library_generation/model/gapic_config.py hermetic_build/common/model/gapic_config.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 from hashlib import sha256
 from typing import Optional
-from library_generation.model.gapic_config import GapicConfig
-from library_generation.model.gapic_inputs import GapicInputs
+from common.model.gapic_config import GapicConfig
+from common.model.gapic_inputs import GapicInputs
 
 
 MAVEN_COORDINATE_SEPARATOR = ":"

library_generation/model/gapic_inputs.py hermetic_build/common/model/gapic_inputs.py

@@ -0,0 +1,3 @@
+black==24.8.0
+parameterized==0.9.0
+PyYAML==6.0.2

library_generation/model/generation_config.py hermetic_build/common/model/generation_config.py

@@ -0,0 +1,109 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --generate-hashes hermetic_build/common/requirements.in
+#
+black==24.8.0 \
+    --hash=sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6 \
+    --hash=sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e \
+    --hash=sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f \
+    --hash=sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018 \
+    --hash=sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e \
+    --hash=sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd \
+    --hash=sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4 \
+    --hash=sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed \
+    --hash=sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2 \
+    --hash=sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42 \
+    --hash=sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af \
+    --hash=sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb \
+    --hash=sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368 \
+    --hash=sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb \
+    --hash=sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af \
+    --hash=sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed \
+    --hash=sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47 \
+    --hash=sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2 \
+    --hash=sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a \
+    --hash=sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c \
+    --hash=sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920 \
+    --hash=sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1
+    # via -r hermetic_build/common/requirements.in
+click==8.1.7 \
+    --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
+    --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
+    # via black
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via black
+packaging==24.1 \
+    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
+    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+    # via black
+parameterized==0.9.0 \
+    --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \
+    --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1
+    # via -r hermetic_build/common/requirements.in
+pathspec==0.12.1 \
+    --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \
+    --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712
+    # via black
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via black
+pyyaml==6.0.2 \
+    --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
+    --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
+    --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \
+    --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \
+    --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \
+    --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \
+    --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \
+    --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \
+    --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \
+    --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \
+    --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \
+    --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \
+    --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \
+    --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \
+    --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \
+    --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \
+    --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \
+    --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \
+    --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \
+    --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \
+    --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \
+    --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \
+    --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \
+    --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \
+    --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \
+    --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \
+    --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \
+    --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \
+    --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \
+    --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \
+    --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \
+    --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \
+    --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \
+    --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \
+    --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \
+    --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \
+    --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \
+    --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \
+    --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \
+    --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \
+    --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \
+    --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \
+    --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \
+    --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \
+    --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \
+    --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \
+    --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \
+    --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \
+    --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \
+    --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \
+    --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \
+    --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
+    --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
+    # via -r hermetic_build/common/requirements.in

library_generation/model/library_config.py hermetic_build/common/model/library_config.py

@@ -0,0 +1,14 @@
+"""
+Package information of library_generation python scripts
+"""
+
+from setuptools import setup
+
+setup(
+    name="common",
+    version="0.1",
+    python_requires=">=3.12",
+    package_dir={
+        "common": ".",
+    },
+)

hermetic_build/common/requirements.in

@@ -13,7 +13,7 @@
 # limitations under the License.
 import unittest
 
-from library_generation.model.gapic_config import GapicConfig
+from common.model.gapic_config import GapicConfig
 
 
 class GapicConfigTest(unittest.TestCase):

hermetic_build/common/requirements.txt

@@ -3,7 +3,7 @@
 from pathlib import Path
 
 from parameterized import parameterized
-from library_generation.model.gapic_inputs import parse
+from common.model.gapic_inputs import parse
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "..", "resources")