chore: bake Java formatter into image (#3271)

In this PR:
- Bake Java formatter into image.

---------

Co-authored-by: cloud-java-bot <[email protected]>

Author: JoeWang1127

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -35,6 +35,7 @@ SHELL [ "/bin/bash", "-c" ]
 ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3
 ARG PROTOC_VERSION=25.5
 ARG GRPC_VERSION=1.67.1
+ARG JAVA_FORMAT_VERSION=1.7
 ENV HOME=/home
 ENV OS_ARCHITECTURE="linux-x86_64"
 
@@ -101,6 +102,11 @@ RUN owl-bot copy-code --version
 RUN chmod -R o+rx ${NODE_PATH}
 RUN ln -sf ${NODE_PATH}/* /usr/local/bin
 
+# download the Java formatter
+ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \
+  "${HOME}"/.library_generation/google-java-format.jar
+RUN chmod 755 "${HOME}"/.library_generation/google-java-format.jar
+
 # allow users to access the script folders
 RUN chmod -R o+rx /src
 

library_generation/DEVELOPMENT.md

@@ -51,9 +51,8 @@ This section explains how to run the entrypoint script
 ## Assumptions made by the scripts
 ### The Hermetic Build's well-known folder
 Located in `${HOME}/.library_generation`, this folder is assumed by the scripts
-to contain the generator JAR. 
-Please note that this is a recent feature and only this jar is expected to be
-there. 
+to contain certain tools.
+
 Developers must make sure this folder is properly configured before running the
 scripts locally.
 Note that this relies on the `HOME` en var which is always defined as per
@@ -73,7 +72,16 @@ The generation scripts will assume the jar is there.
 mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar"
 ```
 
+#### Put the java formatter jar in its well-known location
+
+Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format)
+or [GitHub releases](https://github.com/google/google-java-format/releases).
+Then `mv` the jar into the well-known location of the jar.
+The generation scripts will assume the jar is there.
 
+```shell
+mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar"
+```
 
 ## Installing prerequisites
 

library_generation/README.md

@@ -8,7 +8,6 @@ google-cloud-java) from a configuration file.
 
 - OS: Linux
 - Java runtime environment (8 or above)
-- Apache Maven (used in formatting source code)
 - Python (3.11.6 or above)
 - Docker 
 - Git

library_generation/owlbot/bin/entrypoint.sh

@@ -65,20 +65,14 @@ echo "...done"
 
 # write or restore clirr-ignored-differences.xml
 echo "Generating clirr-ignored-differences.xml..."
-${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
+"${scripts_root}"/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
 echo "...done"
 
 # fix license headers
 echo "Fixing missing license headers..."
 python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
 echo "...done"
 
-# Ensure formatting on all .java files in the repository.
-# Here we manually set the user.home system variable. Unfortunately, Maven
-# user.home inference involves the /etc/passwd file (confirmed empirically),
-# instead of the presumable $HOME env var, which may not work properly
-# when `docker run`ning with the -u flag because we may incur in users
-# not registered in the container's /etc/passwd file
 echo "Reformatting source..."
-mvn fmt:format -Duser.home="${HOME}" -V --batch-mode --no-transfer-progress
+"${scripts_root}"/owlbot/bin/format_source.sh "${scripts_root}"
 echo "...done"

library_generation/owlbot/bin/format_source.sh

@@ -22,9 +22,11 @@ set -e
 
 # Find all the java files relative to the current directory and format them
 # using google-java-format
-list="$(find . -name '*.java' -not -path ".*/samples/snippets/generated/**/*" )"
 scripts_root=$1
-tmpfile=$(mktemp)
+
+source "${scripts_root}/utils/utilities.sh"
+list="$(find . -name '*.java' -not -path ".*/samples/snippets/generated/**/*" )"
+tmp_file=$(mktemp)
 
 for file in $list;
 do
@@ -35,19 +37,11 @@ do
   then
     echo "File skipped formatting: $file"
   else
-   echo $file >> $tmpfile
+   echo $file >> $tmp_file
   fi
 done
 
-# download the google-java-format tool
-if [ ! -f "${scripts_root}/owlbot/google-java-format.jar" ]; then
-  echo 'downloading google-java-format'
-  java_format_version=$(cat "${scripts_root}/configuration/java-format-version")
-  wget -O "${scripts_root}/owlbot/google-java-format.jar" https://repo1.maven.org/maven2/com/google/googlejavaformat/google-java-format/${java_format_version}/google-java-format-${java_format_version}-all-deps.jar
-
-fi
-
-# format the source
-cat $tmpfile | xargs java -jar "${scripts_root}/owlbot/google-java-format.jar" --replace
+# use formatter downloaded in the Dockerfile.
+cat $tmp_file | xargs java -jar "$(get_java_formatter_location)" --replace
 
-rm $tmpfile
+rm $tmp_file

library_generation/test/generate_library_unit_tests.sh

@@ -12,6 +12,7 @@ source "${script_dir}"/test_utilities.sh
 readonly SIMULATED_HOME=$(mktemp -d)
 mkdir "${SIMULATED_HOME}/.library_generation"
 touch "${SIMULATED_HOME}/.library_generation/gapic-generator-java.jar"
+touch "${SIMULATED_HOME}/.library_generation/google-java-format.jar"
 HOME="${SIMULATED_HOME}" source "${script_dir}"/../utils/utilities.sh
 
 # Unit tests

library_generation/test/utilities_unit_tests.py

@@ -303,6 +303,14 @@ def test_prepare_repo_split_repo_success(self):
         self.assertEqual(["misc"], library_path)
         shutil.rmtree(repo_config.output_folder)
 
+    def test_get_java_generator_location_success(self):
+        location = util.sh_util("get_gapic_generator_location")
+        self.assertRegex(location, r"/.library_generation/gapic-generator-java.jar$")
+
+    def test_get_java_formatter_location_success(self):
+        location = util.sh_util("get_java_formatter_location")
+        self.assertRegex(location, r"/.library_generation/google-java-format.jar$")
+
     def __setup_postprocessing_prerequisite_files(
         self,
         combination: int,

library_generation/utils/utilities.sh

@@ -4,6 +4,7 @@ set -eo pipefail
 utilities_script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 # The $HOME variable is always set in the OS env as per POSIX specification.
 GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar"
+JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar"
 
 # Utility functions used in `generate_library.sh` and showcase generation.
 extract_folder_name() {
@@ -137,10 +138,11 @@ get_protoc_version() {
 # and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no
 # download), since the docker image will have downloaded these tools beforehand.
 #
-# For the case of gapic-generator-java, no env var will be exported for the
-# upstream flow. Instead, the jar must be located in the well-known location
-# (${HOME}/.library_generation/gapic-generator-java.jar). More information in
-# `library_generation/DEVELOPMENT.md`
+# For the case of generator and formatter, no env var will be exported for the
+# upstream flow.
+# Instead, the jar must be located in the well-known location
+# (${HOME}/.library_generation/).
+# More information in `library_generation/DEVELOPMENT.md`.
 download_tools() {
   local protoc_version=$1
   local grpc_version=$2
@@ -166,18 +168,11 @@ download_tools() {
     export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}")
   fi
 
-  # Here we check whether the jar is stored in the expected location.
-  # The docker image will prepare the jar in this location. Developers must
-  # prepare their environment by creating
-  # $HOME/.library_generation/gapic_generator_java.jar
-  # This check is meant to ensure integrity of the downstream workflow. (i.e.
-  # ensure the generator wrapper succeeds)
-  if [[ ! -f "${GAPIC_GENERATOR_LOCATION}" ]]; then
-    >&2 echo "File ${GAPIC_GENERATOR_LOCATION} not found in the "
-    >&2 echo "filesystem. Please configure your environment and store the "
-    >&2 echo "generator jar in this location"
-    exit 1
-  fi
+  # Here we check whether required tools is stored in the expected location.
+  # The docker image will prepare jar files in this location.
+  # This check is meant to ensure integrity of the downstream workflow.
+  error_if_not_exists "${GAPIC_GENERATOR_LOCATION}"
+  error_if_not_exists "${JAVA_FORMATTER_LOCATION}"
   popd
 }
 
@@ -380,3 +375,17 @@ download_googleapis_files_and_folders() {
 get_gapic_generator_location() {
   echo "${GAPIC_GENERATOR_LOCATION}"
 }
+
+get_java_formatter_location() {
+  echo "${JAVA_FORMATTER_LOCATION}"
+}
+
+error_if_not_exists() {
+  local required_tool=$1
+  if [[ ! -f "${required_tool}" ]]; then
+    >&2 echo "File ${required_tool} not found in the filesystem. "
+    >&2 echo "Please configure your environment and store the "
+    >&2 echo "required tools in this location."
+    exit 1
+  fi
+}

showcase/scripts/generate_showcase.sh

@@ -55,30 +55,42 @@ if [ ! -d google ];then
   rm -rdf googleapis
 fi
 
-# copy the generator into its well-known location. For more details,
+# copy the generator and formatter into its well-known location.
+# For more details,
 # refer to library_generation/DEVELOPMENT.md
+java_formatter_name="google-java-format.jar"
+java_formatter_version="1.7"
 well_known_folder="${HOME}/.library_generation"
 well_known_generator_jar_location="${well_known_folder}/gapic-generator-java.jar"
+well_known_formatter_jar_location="${well_known_folder}/${java_formatter_name}"
 if [[ ! -d "${well_known_folder}" ]]; then
   mkdir "${well_known_folder}"
 fi
 if [[ -f "${well_known_generator_jar_location}" ]]; then
   echo "replacing well-known generator jar with the latest one"
   rm "${well_known_generator_jar_location}"
 fi
+if [[ -f "${well_known_formatter_jar_location}" ]]; then
+  echo "replacing well-known formatter jar with the latest one"
+  rm "${well_known_formatter_jar_location}"
+fi
 maven_repository="$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)"
 generator_version=$(get_version_from_versions_txt "gapic-generator-java")
-source_jar_path="${maven_repository}/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar"
+generator_jar_path="${maven_repository}/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar"
 
-if [[ ! -f "${source_jar_path}" ]]; then
+if [[ ! -f "${generator_jar_path}" ]]; then
   echo "generator jar not found in its assumed location"
-  echo "in the local repository: ${source_jar_path}"
+  echo "in the local repository: ${generator_jar_path}"
   echo "(did you run mvn install in this repository's root?)"
   exit 1
 fi
 # transfer the snapshot jar into its well-known location
-cp "${source_jar_path}" "${well_known_generator_jar_location}"
-
+cp "${generator_jar_path}" "${well_known_generator_jar_location}"
+# transfer java formatter to its well-known location
+download_from \
+  "https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${java_formatter_version}/google-java-format-${java_formatter_version}-all-deps.jar" \
+  "${java_formatter_name}"
+cp "${java_formatter_name}" "${well_known_formatter_jar_location}"
 gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto"
 
 path_to_generator_parent_pom="${SCRIPT_DIR}/../../gapic-generator-java-pom-parent/pom.xml"