feat: bake gapic-generator-java into the hermetic build docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -12,11 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# install gapic-generator-java in a separate layer so we don't overload the image
+# with the transferred source code and jars
+FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+
+WORKDIR /sdk-platform-java
+COPY . .
+# {x-version-update-start:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.44.1-SNAPSHOT" 
+# {x-version-update-end:gapic-generator-java:current}
+
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
+  "./gapic-generator-java.jar"
+
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
+
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.4
 ARG GRPC_VERSION=1.66.0
@@ -47,7 +62,15 @@ RUN source /src/utils/utilities.sh \
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
-# use python 3.11 (the base image has several python versions; here we define the default one)
+
+# Here we transfer gapic-generator-java from the previous stage.
+# Note that the destination is a well-known location that will be assumed at runtime
+# We hard-code the location string to avoid making it configurable (via ARG) as
+# well as to avoid it making it overridable at runtime (via ENV).
+COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
+RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+
+#  use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3

.github/workflows/ci.yaml

@@ -232,6 +232,17 @@ jobs:
       - name: Showcase golden tests
         working-directory: showcase
         run: |
+          # The golden test directly calls
+          # library_generation/generate_library.sh, which expects the jar to be
+          # located in its well-known location. More info in
+          # library_generation/DEVELOPMENT.md
+          # Here we prepare the jar in such location
+          generator_version=$(grep "gapic-generator-java:" "../versions.txt" \
+            | cut -d: -f3) # the 3rd field is the snapshot version
+          mkdir -p "${HOME}/.library_generation"
+          cp \
+            "${HOME}/.m2/repository/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar" \
+            "${HOME}/.library_generation/gapic-generator-java.jar"
           mvn test \
             -P enable-golden-tests \
             --batch-mode \

.kokoro/presubmit/downstream-compatibility-spring.sh

@@ -34,8 +34,8 @@ git clone "https://github.com/GoogleCloudPlatform/spring-cloud-gcp.git" --depth=
 update_all_poms_dependency "spring-cloud-gcp" "gapic-generator-java-bom" "${GAPIC_GENERATOR_VERSION}"
 
 # Install spring-cloud-gcp modules
-pushd spring-cloud-gcp/spring-cloud-generator
-../mvnw \
+pushd spring-cloud-gcp
+./mvnw \
   -U \
   --batch-mode \
   --no-transfer-progress \
@@ -47,10 +47,16 @@ pushd spring-cloud-gcp/spring-cloud-generator
 
 
 # Generate showcase autoconfig
+pushd spring-cloud-generator
+# The script is not executable for non-owners. Here we manually chmod it.
+# TODO(diegomarquezp): remove this line after
+# https://github.com/GoogleCloudPlatform/spring-cloud-gcp/pull/3183 is merged
+chmod 755 ./scripts/generate-showcase.sh
 ./scripts/generate-showcase.sh
 pushd showcase/showcase-spring-starter
 mvn verify
 popd # showcase/showcase-spring-starter
 
-popd # spring-cloud-gcp/spring-cloud-generator
+popd # spring-cloud-generator
+popd # spring-cloud-gcp
 popd # gapic-generator-java/target

library_generation/DEVELOPMENT.md

@@ -48,32 +48,36 @@ Although the scripts are designed to be run in a Docker container, you can also
 run them directly. This section explains how to run the entrypoint script
 (`library_generation/cli/entry_point.py`).
 
-## Installing prerequisites
+## Assumptions made by the scripts
+### The Hermetic Build's well-known folder
+Located in `${HOME}/.library_generation`, this folder is assumed by the scripts
+to contain the generator JAR. Please note that this is a recent feature and only
+this jar is expected to be there. Soon enough, more binaries such as the gRPC
+and protoc plugins will also be stored here. Developers must make sure
+this folder is properly configured before running the scripts locally.
+Note that this relies on the `HOME` en var which is always
+defined as per
+[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html).
 
-In order to run the generation scripts directly, there are a few tools we
-need to install beforehand.
+#### Put the gapic-generator-java jar in its well-known location
 
-### Install synthtool
+Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip
+-Dcheckstyle.skip`. This will generate a jar located in
+`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar`
 
-It requires python 3.x to be installed.
-You will need to specify a committish of the synthtool repo in order to have
-your generation results matching exactly what the docker image would produce.
-You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in
-`.cloudbuild/library_generation/library_generation.Dockerfile`.
+Then `mv` the jar into the well-known location of the jar. The
+generation scripts will assume the jar is there.
 
-```bash
-# obtained from .cloudbuild/library_generation/library_generation.Dockerfile
-export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
+```shell
+mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar"
 ```
 
-```bash
-git clone https://github.com/googleapis/synthtool
-cd synthtool
-git checkout "${SYNTHTOOL_COMMITTISH}"
-python -m pip install --require-hashes -r requirements.txt
-python -m pip install --no-deps -e .
-python -m synthtool --version
-```
+
+
+## Installing prerequisites
+
+In order to run the generation scripts directly, there are a few tools we
+need to install beforehand.
 
 ### Install the owl-bot CLI
 
@@ -93,6 +97,7 @@ owl-bot copy-code --version
 The key step is `npm link`, which will make the command available in you current
 shell session.
 
+
 ## Running the script
 The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
 update the target repository with the latest changes starting from the

library_generation/gapic-generator-java-wrapper

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-
 set -e
+wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${wrapper_dir}/utils/utilities.sh"
 
 # Wrap gapic-generator-java.jar because protoc requires the plugin to be executable.
-exec java -classpath "gapic-generator-java-${gapic_generator_version}.jar" com.google.api.generator.Main
+exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main

library_generation/generate_composed_library.py

@@ -131,7 +131,6 @@ def __construct_tooling_arg(config: GenerationConfig) -> List[str]:
     :return: arguments containing tooling versions
     """
     arguments = []
-    arguments += util.create_argument("gapic_generator_version", config)
     arguments += util.create_argument("grpc_version", config)
     arguments += util.create_argument("protoc_version", config)
 

library_generation/generate_library.sh

@@ -14,12 +14,6 @@ case $key in
     destination_path="$2"
     shift
     ;;
-  --gapic_generator_version)
-    gapic_generator_version="$2"
-    # export this variable so that it can be used in gapic-generator-java-wrapper.sh
-    export gapic_generator_version
-    shift
-    ;;
   --protoc_version)
     protoc_version="$2"
     shift
@@ -77,17 +71,12 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
-if [ -z "${gapic_generator_version}" ]; then
-  echo 'missing required argument --gapic_generator_version'
-  exit 1
-fi
-
 if [ -z "${protoc_version}" ]; then
-  protoc_version=$(get_protoc_version "${gapic_generator_version}")
+  protoc_version=$(get_protoc_version)
 fi
 
 if [ -z "${grpc_version}" ]; then
-  grpc_version=$(get_grpc_version "${gapic_generator_version}")
+  grpc_version=$(get_grpc_version)
 fi
 
 if [ -z "${proto_only}" ]; then
@@ -185,7 +174,7 @@ esac
 # download gapic-generator-java, protobuf and grpc plugin.
 # the download_tools function will create the environment variables "protoc_path"
 # and "grpc_path", to be used in the protoc calls below.
-download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}"
+download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}"
 ##################### Section 1 #####################
 # generate grpc-*/
 #####################################################