build: make tests run faster (#4205)

* build: make tests run faster

Author: sofisl

.github/sync-repo-settings.yaml

@@ -34,9 +34,9 @@ branchProtectionRules:
     - "samples-presubmit-node14-with-credentials (long-door-651)"
     - "samples-presubmit-node14 (long-door-651)"
     - "units (14)"
-    - "unit-presubmit-node16 (long-door-651)"
-    - "unit-presubmit-node18 (long-door-651)"
-    - "unit-presubmit-node12 (long-door-651)"
+    - "units (12)"
+    - "units (16)"
+    - "units (18)"
 # List of explicit permissions to add (additive only)
 permissionRules:
     # Team slug to add to repository permissions

.github/workflows/continuous.yaml

@@ -11,9 +11,9 @@ jobs:
         node: [14]
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: pnpm/action-setup@v2
         with:
-          node-version: ${{ matrix.node }}
+          version: 7.32.2
       - run: node --version
       - run: ci/run_conditional_tests.sh
         name: Check lint
@@ -24,12 +24,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [14]
+        node: [12, 14, 16, 18]
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: pnpm/action-setup@v2
         with:
-          node-version: ${{ matrix.node }}
+          version: ^6.24.1
       - run: node --version
       - run: ci/run_conditional_tests.sh
         name: Run unit tests

.github/workflows/presubmit.yaml

@@ -9,9 +9,9 @@ jobs:
         node: [14]
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: pnpm/action-setup@v2
         with:
-          node-version: ${{ matrix.node }}
+          version: 7.32.2
       - run: node --version
       - run: ci/run_conditional_tests.sh
         name: Check lint
@@ -22,12 +22,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [14]
+        node: [12, 14, 16, 18]
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: pnpm/action-setup@v2
         with:
-          node-version: ${{ matrix.node }}
+          version: ^6.24.1
       - run: node --version
       - run: ci/run_conditional_tests.sh
         name: Run unit tests

.gitignore

@@ -9,3 +9,5 @@ packages/*/CONTRIBUTORS
 packages/*/COPYING
 package-lock.json
 __pycache__
+packages/*/pnpm-lock.yaml
+containers/*/pnpm-lock.yaml

ci/Dockerfile

@@ -31,16 +31,19 @@ RUN curl https://github.com/googleapis/repo-automation-bots/releases/download/fl
     -o /bin/flakybot -s -L \
     && chmod +x /bin/flakybot
 
-FROM node:${NODE_VERSION}-alpine
+ENV PNPM_VERSION=7.32.2
+RUN curl https://get.pnpm.io/install.sh | ENV="$HOME/.shrc" SHELL="$(which sh)" sh -
 
-COPY --from=build /bin/flakybot /bin/flakybot
+FROM node:${NODE_VERSION}-alpine
 
 # Hack for not found error with flakybot
 RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
 
 COPY --from=build /usr/local/gcloud /usr/local/gcloud
 COPY --from=build /bin/flakybot /bin/flakybot
+COPY --from=build /root/.local/share/pnpm /root/.local/share/pnpm
 
 RUN apk add --no-cache git bash python3
 
-ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+ENV PNPM_HOME=/root/.local/share/pnpm
+ENV PATH=$PNPM_HOME:$PATH

ci/cloudbuild.yaml

@@ -35,6 +35,11 @@ steps:
       '--push-retry=3',
       '--image-fs-extract-retry=3'
     ]
+  - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+    entrypoint: "bash"
+    args: ["ci/deletecloudbuild.sh"]
+    env:
+    - 'REF_NAME=${REF_NAME}'
   - name: gcr.io/${PROJECT_ID}/google-cloud-node-${_NODE_VERSION}
     id: "run-tests"
     timeout: 32400s

ci/deletecloudbuild.sh

@@ -0,0 +1,19 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+now=$(date -u -d "-15 min" +"%Y-%m-%dT%H:%M:%SZ")
+gcloud builds list --format="value(id)" --filter="substitutions.REF_NAME="$REF_NAME" AND status="WORKING" AND create_time<"$now"" | while IFS= read -r line ; do
+    echo "gcloud builds cancel "$line""
+    gcloud builds cancel "$line"
+done

ci/export/unit-presubmit-node12.yaml

@@ -46,12 +46,4 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do
     gcloud beta builds triggers export "samples-nightly-node${NODE_VERSION}" --destination "ci/export/samples-nightly-node${NODE_VERSION}.yaml"
     echo "exporting presubmit build for node${NODE_VERSION} with credentials"
     gcloud beta builds triggers export "samples-presubmit-node${NODE_VERSION}-with-credentials" --destination "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml"
-done
-
-# Since we don't want to run nightly, presubmit, and continuous for node 16 and node 18, adding these outside of the loop
-echo "exporting presubmit unit build for node12 unit tests"
-gcloud beta builds triggers export "unit-presubmit-node12" --destination "ci/export/unit-presubmit-node12.yaml"
-echo "exporting presubmit unit build for node16 unit tests"
-gcloud beta builds triggers export "unit-presubmit-node16" --destination "ci/export/unit-presubmit-node16.yaml"
-echo "exporting presubmit unit build for node18 unit tests"
-gcloud beta builds triggers export "unit-presubmit-node18" --destination "ci/export/unit-presubmit-node18.yaml"
+done

ci/export/unit-presubmit-node16.yaml

@@ -46,12 +46,4 @@ for NODE_VERSION in ${NODE_VERSIONS[@]}; do
     gcloud beta builds triggers import --source "ci/export/samples-nightly-node${NODE_VERSION}.yaml"
     echo "importing presubmit build for node${NODE_VERSION} with credentials"
     gcloud beta builds triggers import --source "ci/export/samples-presubmit-node${NODE_VERSION}-with-credentials.yaml"
-done
-
-# Since we don't want to run nightly, and continuous for node 16 and node 18, adding these outside of the loop
-echo "importing presubmit unit build for node12 unit tests"
-gcloud beta builds triggers import --source "ci/export/unit-presubmit-node12.yaml"
-echo "importing presubmit unit build for node16 unit tests"
-gcloud beta builds triggers import --source "ci/export/unit-presubmit-node16.yaml"
-echo "importing presubmit unit build for node18 unit tests"
-gcloud beta builds triggers import --source "ci/export/unit-presubmit-node18.yaml"
+done

ci/export/unit-presubmit-node18.yaml

@@ -84,7 +84,7 @@ for subdir in ${subdirs[@]}; do
             if [[ "${changed}" -eq 0 ]]; then
                 echo "no change detected in ${d}, skipping"
             else
-                if [[ "${TEST_TYPE}" == "system" ]]; then
+                if [[ "${TEST_TYPE}" == "system" ]] || [[ "${TEST_TYPE}" == "lint" ]] || [[ "${TEST_TYPE}" == "units" ]]; then
                     echo "change detected in ${d} for system test"
                     should_test=true
                 elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
@@ -97,7 +97,7 @@ for subdir in ${subdirs[@]}; do
             fi
         else
             # If GIT_DIFF_ARG is empty, run all the tests.
-            if [[ "${TEST_TYPE}" == "system" ]]; then
+            if [[ "${TEST_TYPE}" == "system" ]] || [[ "${TEST_TYPE}" == "lint" ]] || [[ "${TEST_TYPE}" == "units" ]]; then
                 echo "run system test for ${d}"
                 should_test=true
             elif [[ "${tests_with_credentials[*]}" =~ "${d}" ]] && [[ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
@@ -118,6 +118,8 @@ for subdir in ${subdirs[@]}; do
             set -e
             if [ ${ret} -ne 0 ]; then
                 RETVAL=${ret}
+                # Since there are so many APIs, we should exit early if there's an error
+                break
             fi
             popd
         fi

ci/export_triggers.sh

@@ -41,36 +41,37 @@ if [ ${BUILD_TYPE} != "presubmit" ]; then
 fi
 
 # Install dependencies
-if [[ "${tests_without_node12[*]}" =~ "${PROJECT}" ]] && [[ ${NODE_VERSION} == "12.22.12" ]]; then
+if [[ "${tests_without_node12[*]}" =~ "${PROJECT}" ]] && [[ ${NODE_VERSION} == "12.22.12" ]] && [[ ${TEST_TYPE} != "lint" ]]; then
     echo "not running tests for Node 12"
 elif [[ ${NODE_VERSION} == "12.22.12" ]]; then
     echo "npm install --unsafe-perm --ignore-scripts --engine-strict --only=prod; npm install --unsafe-perm"
     npm install --unsafe-perm --ignore-scripts --engine-strict --only=prod; npm install --unsafe-perm
 else
-    echo "npm install --ignore-scripts --engine-strict --omit=dev; npm install"
-    npm install --ignore-scripts --engine-strict --omit=dev; npm install
+    echo "pnpm install --ignore-scripts --engine-strict --prod; pnpm install"
+    pnpm install --ignore-scripts --engine-strict --prod; pnpm install
 fi
 
 retval=0
 
 set +e
 case ${TEST_TYPE} in
 lint)
-    npm run lint
+    pnpm prelint
+    pnpm lint
     retval=$?
     ;;
 samples)
-    npm run samples-test
+    pnpm samples-test
     retval=$?
     ;;
 system)
-    npm run system-test
+    pnpm system-test
     retval=$?
     ;;
 units)
     # Skipping Node 12 tests for projects outside of gapic autogenerated libraries that don't need to run on Node 12
     if ! [[ "${tests_without_node12[*]}" =~ "${PROJECT}" ]] || [[ ${NODE_VERSION} != "12.22.12" ]]; then
-        npm test
+        pnpm test
         retval=$?
     fi
     ;;

ci/import_triggers.sh

@@ -20,7 +20,7 @@
   },
   "homepage": "https://github.com/googleapis/google-cloud-node#readme",
   "dependencies": {
-    "@octokit/rest": "^19.0.0"
+    "@octokit/rest": "19.0.0"
   },
   "devDependencies": {
     "mocha": "^9.0.0",

ci/run_conditional_tests.sh

@@ -30,17 +30,18 @@
   ],
   "scripts": {
     "clean": "gts clean",
-    "compile": "tsc -p . && cp -r protos build/ && minifyProtoJson",
+    "compile": "tsc -p . && cp -r protos build/",
     "compile-protos": "compileProtos src",
     "docs": "jsdoc -c .jsdoc.js",
     "predocs-test": "npm run docs",
     "docs-test": "linkinator docs",
     "fix": "gts fix",
     "lint": "gts check",
-    "prelint": "cd samples; npm link ../; npm i",
     "prepare": "npm run compile-protos && npm run compile",
-    "samples-test": "npm run compile && cd samples/ && npm link ../ && npm i && npm test",
-    "system-test": "npm run compile && c8 mocha build/system-test",
+    "prelint": "cd samples; npm link ../; npm i",
+    "postpack": "minifyProtoJson",
+    "samples-test": "cd samples/ && npm link ../ && npm i && npm test",
+    "system-test": "c8 mocha build/system-test",
     "test": "c8 mocha build/test"
   },
   "dependencies": {
@@ -58,6 +59,7 @@
     "jsdoc-fresh": "^2.0.0",
     "jsdoc-region-tag": "^2.0.0",
     "linkinator": "^4.0.0",
+    "long": "^5.2.3",
     "mocha": "^9.2.2",
     "null-loader": "^4.0.0",
     "pack-n-play": "^1.0.0-2",
@@ -69,5 +71,4 @@
     "node": ">=12.0.0"
   },
   "homepage": "https://github.com/googleapis/google-cloud-node/tree/main/packages/google-analytics-admin"
-}
-
+}

ci/run_single_test.sh

@@ -1,3 +1,4 @@
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -29,6 +30,6 @@ const cwd = path.join(__dirname, '..');
 describe('Quickstart', async () => {
   it('should run quickstart', async () => {
     const output = execSync('node quickstart.js', {cwd});
-    assert(output != null);
+    assert(output !== null);
   });
 });

containers/node-bootstrap-container/package.json

@@ -30,17 +30,18 @@
   ],
   "scripts": {
     "clean": "gts clean",
-    "compile": "tsc -p . && cp -r protos build/ && minifyProtoJson",
+    "compile": "tsc -p . && cp -r protos build/",
     "compile-protos": "compileProtos src",
     "docs": "jsdoc -c .jsdoc.js",
     "predocs-test": "npm run docs",
     "docs-test": "linkinator docs",
     "fix": "gts fix",
     "lint": "gts check",
-    "prelint": "cd samples; npm link ../; npm i",
     "prepare": "npm run compile-protos && npm run compile",
-    "samples-test": "npm run compile && cd samples/ && npm link ../ && npm i && npm test",
-    "system-test": "npm run compile && c8 mocha build/system-test",
+    "prelint": "cd samples; npm link ../; npm i",
+    "postpack": "minifyProtoJson",
+    "samples-test": "cd samples/ && npm link ../ && npm i && npm test",
+    "system-test": "c8 mocha build/system-test",
     "test": "c8 mocha build/test"
   },
   "dependencies": {
@@ -56,6 +57,7 @@
     "jsdoc-fresh": "^2.0.0",
     "jsdoc-region-tag": "^2.0.0",
     "linkinator": "^4.0.0",
+    "long": "^5.2.3",
     "mocha": "^9.2.2",
     "null-loader": "^4.0.0",
     "pack-n-play": "^1.0.0-2",
@@ -69,5 +71,4 @@
     "node": ">=12.0.0"
   },
   "homepage": "https://github.com/googleapis/google-cloud-node/tree/main/packages/google-analytics-data"
-}
-
+}