feat: hermetic build scripts to use a single output/generation folder (#1987)

* move generation tools and generated source to single folder

* script fixes

* capture exit code from failing script

* remove accidental file

* wip: fix showcase test

* post rebase fixes

* adjust showcase failure handling

* update readme

* fix output path in IT

* download googleapis in output folder

* Update library_generation/README.md

Co-authored-by: Joe Wang <[email protected]>

* remove unrelated os detection function

* use pushd instead of cd in gen script output cleanup

* correct destination_path documentation

* Update library_generation/README.md

Co-authored-by: Joe Wang <[email protected]>

* fix pushd in generate_library.sh

* ignore output folder

* Update library_generation/README.md

Co-authored-by: Joe Wang <[email protected]>

* Update library_generation/README.md

Co-authored-by: Joe Wang <[email protected]>

* spit output to stdout

* centralize output folder computation

* label popped folders

* prevent wrong version unit test from exiting the test suite

* resolve output dir using pwd

---------

Co-authored-by: Joe Wang <[email protected]>

Author: diegomarquezp

.gitignore

@@ -15,4 +15,9 @@ target/
 # Vscode Settings
 .vscode/settings.json
 
-*.iml
+*.iml
+
+# library generation
+output/
+library_generation/output/
+showcase/scripts/output/

library_generation/README.md

@@ -7,37 +7,38 @@ The script, `generate_library.sh`, allows you to generate a GAPIC client library
 Use Linux environment and install java runtime environment (8 or above).
 
 ## Prerequisite
-Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to the current
-working directory (refers as `$cwd`, a directory contains `generate_library.sh`).
+Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to an `output`
+directory located in the working directory, referred as `$cwd`
+(for example, `library_generation/output` if planning to call from the same folder).
 The directory structure should be the same as import statements in protos.
 
-For example, we want to generate from `folder1/folder2/protoA`, so `proto_path` 
-should be set to `folder1/folder2` (a relative path from `$cwd`). 
+For example, you want to generate from `folder1/folder2/protoA`, so `proto_path` 
+should be set to `folder1/folder2` (a relative path from `output`). 
 protoA imports protoB as `folder3/folder4/protoB`, then there should 
-be `folder3/folder4` (containing protoB) in `$cwd`.
+be `folder3/folder4` (containing protoB) in `output`.
 
 In order to generate a GAPIC library, you need to pull `google/` from [googleapis](https://github.com/googleapis/googleapis)
-and put it into `$cwd` since protos in `google/` are likely referenced by 
+and put it into `output` since protos in `google/` are likely referenced by 
 protos from which the library are generated.
 
 ## Parameters to run `generate_library.sh`
 
 You need to run the script with the following parameters.
 
 ### proto_path
-A directory in `$cwd` and copy proto files into it. 
-The absolute path of `proto_path` is `$cwd/$proto_path`. 
+A directory in `$cwd/output` and copy proto files into it. 
+The absolute path of `proto_path` is `$cwd/output/$proto_path`. 
 
 Use `-p` or `--proto_path` to specify the value.
 
 ### destination_path 
-A directory within `$cwd`. 
+A directory within `$cwd/output`. 
 This is the path in which the generated library will reside. 
-The absolute path of `destination_path` is `$cwd/$destination_path`. 
+The absolute path of `destination_path` is `$cwd/output/$destination_path`. 
 
 Use `-d` or `--destination_path` to specify the value.
 
-Note that you do not need to create `$detination_path` beforehand.
+Note that you do not need to create `$destination_path` beforehand.
 
 The directory structure of the generated library is
 ```

library_generation/generate_library.sh

@@ -54,8 +54,9 @@ shift # past argument or value
 done
 
 script_dir=$(dirname "$(readlink -f "$0")")
-# source utility functions
 source "${script_dir}"/utilities.sh
+output_folder="$(get_output_folder)"
+# source utility functions
 
 if [ -z "${protobuf_version}" ]; then
   protobuf_version=$(get_protobuf_version "${gapic_generator_version}")
@@ -81,15 +82,17 @@ if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
 
-mkdir -p "${destination_path}"
+
+mkdir -p "${output_folder}/${destination_path}"
 ##################### Section 0 #####################
 # prepare tooling
 #####################################################
 # the order of services entries in gapic_metadata.json is relevant to the
 # order of proto file, sort the proto files with respect to their name to
 # get a fixed order.
-proto_files=$(find "${proto_path}" -type f  -name "*.proto" | sort)
 folder_name=$(extract_folder_name "${destination_path}")
+pushd "${output_folder}"
+proto_files=$(find "${proto_path}" -type f  -name "*.proto" | sort)
 # download gapic-generator-java, protobuf and grpc plugin.
 download_tools "${gapic_generator_version}" "${protobuf_version}" "${grpc_version}" "${os_architecture}"
 ##################### Section 1 #####################
@@ -156,9 +159,11 @@ for proto_src in ${proto_files}; do
   mkdir -p "${destination_path}/proto-${folder_name}/src/main/proto"
   rsync -R "${proto_src}" "${destination_path}/proto-${folder_name}/src/main/proto"
 done
+popd # output_folder
 ##################### Section 4 #####################
 # rm tar files
 #####################################################
-cd "${destination_path}"
+pushd "${output_folder}/${destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
+popd
 set +x

library_generation/test/generate_library_integration_test.sh

@@ -14,6 +14,10 @@ set -xeo pipefail
 
 # defaults
 googleapis_gen_url="[email protected]:googleapis/googleapis-gen.git"
+script_dir=$(dirname "$(readlink -f "$0")")
+source "${script_dir}/../utilities.sh"
+library_generation_dir="${script_dir}"/..
+output_folder="$(get_output_folder)"
 
 while [[ $# -gt 0 ]]; do
 key="$1"
@@ -38,14 +42,13 @@ esac
 shift # past argument or value
 done
 
-script_dir=$(dirname "$(readlink -f "$0")")
-source "${script_dir}/../utilities.sh"
-library_generation_dir="${script_dir}"/..
-cd "${library_generation_dir}"
+mkdir -p "${output_folder}"
+pushd "${output_folder}"
 # checkout the master branch of googleapis/google (proto files) and WORKSPACE
 echo "Checking out googlapis repository..."
 sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning"
-cd googleapis
+pushd googleapis
+cp -r google "${output_folder}"
 # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
 gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
 echo "The version of gapic-generator-java is ${gapic_generator_version}."
@@ -60,6 +63,8 @@ rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}
 include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}")
 echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}."
 # generate GAPIC client library
+popd
+popd
 echo "Generating library from ${proto_path}, to ${destination_path}..."
 "${library_generation_dir}"/generate_library.sh \
 -p "${proto_path}" \
@@ -74,19 +79,19 @@ echo "Generating library from ${proto_path}, to ${destination_path}..."
 echo "Generate library finished."
 echo "Checking out googleapis-gen repository..."
 
+pushd "${output_folder}"
 sparse_clone "${googleapis_gen_url}" "${proto_path}"
 
 echo "Compare generation result..."
 RESULT=0
-diff -r "googleapis-gen/${proto_path}/${destination_path}" "${destination_path}" -x "*gradle*" || RESULT=$?
+diff -r "googleapis-gen/${proto_path}/${destination_path}" "${output_folder}/${destination_path}" -x "*gradle*" || RESULT=$?
 
 if [ ${RESULT} == 0 ] ; then
   echo "SUCCESS: Comparison finished, no difference is found."
 else
   echo "FAILURE: Differences found."
 fi
 
-cd ..
-rm -rf googleapis
+popd # output_folder
 
 exit ${RESULT}

library_generation/test/generate_library_unit_tests.sh

@@ -181,15 +181,14 @@ generate_library_failed_with_invalid_generator_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  bash "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
     -d ../"${destination}" \
     --gapic_generator_version 1.99.0 \
     --protobuf_version 23.2 \
     --grpc_version 1.55.1 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"
@@ -199,15 +198,14 @@ generate_library_failed_with_invalid_protobuf_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  bash "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
     -d ../"${destination}" \
     --gapic_generator_version 2.24.0 \
     --protobuf_version 22.99 \
     --grpc_version 1.55.1 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"
@@ -217,14 +215,13 @@ generate_library_failed_with_invalid_grpc_version() {
   local destination="google-cloud-alloydb-v1-java"
   local res=0
   cd "${script_dir}/resources"
-  $("${script_dir}"/../generate_library.sh \
+  bash "${script_dir}"/../generate_library.sh \
     -p google/cloud/alloydb/v1 \
-    -d ../"${destination}" \
+    -d ../output/"${destination}" \
     --gapic_generator_version 2.24.0 \
     --grpc_version 0.99.0 \
     --transport grpc+rest \
-    --rest_numeric_enums true \
-    --os_architecture "$(__get_os_architecture)") || res=$?
+    --rest_numeric_enums true || res=$?
   assertEquals 1 $((res))
   # still need to clean up potential downloaded tooling.
   cleanup "${destination}"

library_generation/utilities.sh

@@ -2,6 +2,7 @@
 
 set -xeo pipefail
 
+
 # private functions that should not be called outside this file.
 
 # Used to obtain configuration values from a bazel BUILD file
@@ -157,13 +158,15 @@ get_protobuf_version() {
 }
 
 download_tools() {
+  pushd "${output_folder}"
   local gapic_generator_version=$1
   local protobuf_version=$2
   local grpc_version=$3
   local os_architecture=$4
   download_generator "${gapic_generator_version}"
   download_protobuf "${protobuf_version}" "${os_architecture}"
   download_grpc_plugin "${grpc_version}" "${os_architecture}"
+  popd
 }
 
 download_generator() {
@@ -197,7 +200,7 @@ download_protobuf() {
     rm "protobuf-${protobuf_version}.zip"
   fi
 
-  protoc_path=protobuf-${protobuf_version}/bin
+  protoc_path="${output_folder}/protobuf-${protobuf_version}/bin"
 }
 
 download_grpc_plugin() {
@@ -305,13 +308,13 @@ sparse_clone() {
   clone_dir=$(basename "${repo_url%.*}")
   rm -rf "${clone_dir}"
   git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}"
-  cd "${clone_dir}"
+  pushd "${clone_dir}"
   if [ -n "${commitish}" ]; then
     git checkout "${commitish}"
   fi
   git sparse-checkout set --no-cone ${paths}
   git checkout
-  cd ..
+  popd
 }
 
 # takes a versions.txt file and returns its version
@@ -322,6 +325,13 @@ get_version_from_versions_txt() {
   echo "${version}"
 }
 
+# gets the output folder where all sources and dependencies will be located. It
+# relies on utilities_script_dir which points to the same location as
+# `generate_library.sh`
+get_output_folder() {
+  echo "$(pwd)/output"
+}
+
 detect_os_architecture() {
   local os_type
   local os_architecture

showcase/scripts/generate_showcase.sh

@@ -11,6 +11,7 @@ source "${lib_gen_scripts_dir}/utilities.sh"
 readonly perform_cleanup=$1
 
 cd "${SCRIPT_DIR}"
+mkdir -p "${SCRIPT_DIR}/output"
 
 # clone gapic-showcase
 if [ ! -d schema ]; then
@@ -23,7 +24,7 @@ if [ ! -d schema ]; then
   showcase_version=$(grep -e '<gapic-showcase.version>' "${SCRIPT_DIR}/../gapic-showcase/pom.xml" | cut -d'>' -f2 | cut -d'<' -f1)
     sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}"
   cd gapic-showcase
-  mv schema ..
+  mv schema ../output
   cd ..
   rm -rdf gapic-showcase
 fi
@@ -32,7 +33,7 @@ if [ ! -d google ];then
     rm -rdf googleapis
   fi
   sparse_clone https://github.com/googleapis/googleapis.git "WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/longrunning google/iam/v1 google/cloud/location google/type"
-  mv googleapis/google .
+  mv googleapis/google output
   rm -rdf googleapis
 fi
 
@@ -41,20 +42,20 @@ ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-
 rest_numeric_enums="false"
 transport="grpc+rest"
 include_samples="false"
-rm -rdf showcase-output
-mkdir showcase-output
+rm -rdf output/showcase-output
+mkdir output/showcase-output
 set +e
 bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \
   --proto_path "schema/google/showcase/v1beta1" \
   --destination_path "showcase-output" \
   --gapic_generator_version "${ggj_version}" \
   --rest_numeric_enums "${rest_numeric_enums}" \
   --include_samples "${include_samples}" \
-  --transport "${transport}" &> out
+  --transport "${transport}" 
 
 exit_code=$?
 if [ "${exit_code}" -ne 0 ]; then
-  rm -rdf showcase-output
+  rm -rdf output
   exit "${exit_code}"
 fi
 set +x

showcase/scripts/showcase_utilities.sh

@@ -3,5 +3,5 @@
 function cleanup {
   script_dir=$1
   cd "${script_dir}"
-  rm -rdf gapic-generator-java* google schema protobuf-* protoc-gen-grpc-java* showcase-output out
+  rm -rdf output gapic-generator-java*
 }

showcase/scripts/verify.sh

@@ -12,16 +12,26 @@ readonly REPLACE_SOURCE='false'
 readonly PERFORM_CLEANUP='false'
 bash "${SCRIPT_DIR}/generate_components.sh" "${REPLACE_SOURCE}" "${PERFORM_CLEANUP}"
 
-# compare with proto library
 PROTO_PROJECT_DIR='proto-gapic-showcase-v1beta1'
-diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/showcase-output/proto-showcase-output/src"
-
-# compare with grpc library
 GRPC_PROJECT_DIR='grpc-gapic-showcase-v1beta1'
-diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/showcase-output/grpc-showcase-output/src/main/java/com"
-
-# compare with gapic library
 GAPIC_PROJECT_DIR='gapic-showcase'
-diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/showcase-output/gapic-showcase-output/src/main/java"
+
+{
+  # compare with proto library
+  (diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/output/showcase-output/proto-showcase-output/src") &&
+
+  # compare with grpc library
+  (diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/output/showcase-output/grpc-showcase-output/src/main/java/com") &&
+
+  # compare with gapic library
+  (diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/output/showcase-output/gapic-showcase-output/src/main/java") 
+} || {
+  failure="true"
+}
 
 cleanup $SCRIPT_DIR
+
+if [ "${failure}" == "true" ]; then
+  exit 1
+fi
+