chore: use new parallel build feature of jsii (#4367)

Improve build times by using the new "build-all-at-once" feature of
jsii-pacmak (for the Java build).

Goes together with aws/jsii#849.

Also parallelize the "does this package exist on NPM" checks.

Author: rix0rrr

pack.sh

@@ -2,24 +2,43 @@
 # Runs "npm package" in all modules. This will produce a "dist/" directory in each module.
 # Then, calls pack-collect.sh to merge all outputs into a root ./pack directory, which is
 # later read by bundle-beta.sh.
-set -e
+set -eu
 export PATH=$PWD/node_modules/.bin:$PATH
 export NODE_OPTIONS="--max-old-space-size=4096 ${NODE_OPTIONS:-}"
 root=$PWD
 
+TMPDIR=${TMPDIR:-$(dirname $(mktemp -u))}
 distdir="$PWD/dist"
 rm -fr ${distdir}
 mkdir -p ${distdir}
 
-scopes=$(lerna ls 2>/dev/null | grep -v "(private)" | cut -d" " -f1 | xargs -n1 -I{} echo "--scope {}" | tr "\n" " ")
+# Split out jsii and non-jsii packages. Jsii packages will be built all at once.
+# Non-jsii packages will be run individually.
+echo "Collecting package list..." >&2
+scripts/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt
 
-# Run the "cdk-package" script in all modules. For jsii modules, this invokes jsii-pacmak which generates and builds multi-language
-# outputs. For non-jsii module, it will just run "npm pack" and place the output in dist/npm
-# (which is similar to how pacmak outputs it).
-lerna run ${scopes} --sort --concurrency=1 --stream package
+# Return lerna scopes from a package list
+function lerna_scopes() {
+  while [[ "${1:-}" != "" ]]; do
+    echo "--scope $1 "
+    shift
+  done
+}
+
+echo "Packaging jsii modules" >&2
+
+# Jsii packaging (all at once using jsii-pacmak)
+jsii-pacmak \
+  --verbose \
+  --outdir $distdir/ \
+  $(cat $TMPDIR/jsii.txt)
+
+# Non-jsii packaging, which means running 'package' in every individual
+# module and rsync'ing the result to the shared dist directory.
+echo "Packaging non-jsii modules" >&2
+lerna run $(lerna_scopes $(cat $TMPDIR/nonjsii.txt)) --sort --concurrency=1 --stream package
 
-# Collect dist/ from all modules into the root dist/
-for dir in $(find packages -name dist | grep -v node_modules); do
+for dir in $(find packages -name dist | grep -v node_modules | grep -v run-wrappers); do
   echo "Merging ${dir} into ${distdir}"
   rsync -av $dir/ ${distdir}/
 done

package.json

@@ -17,6 +17,7 @@
     "fs-extra": "^8.1.0",
     "jest": "^24.9.0",
     "jsii-diff": "^0.20.0",
+    "jsii-pacmak": "^0.20.0",
     "lerna": "^3.18.3",
     "nodeunit": "^0.11.3",
     "nyc": "^14.1.1",

scripts/check-api-compatibility.sh

@@ -9,38 +9,56 @@ package_name() {
     node -pe "require('$1/package.json').name"
 }
 
+# Determine whether an NPM package exists on NPM
+#
+# Doesn't use 'npm view' as that is slow. Direct curl'ing npmjs is better
+package_exists_on_npm() {
+    curl -I 2>/dev/null https://registry.npmjs.org/$1 | head -n 1 | grep 200 >/dev/null
+}
+
+
 #----------------------------------------------------------------------
 
-echo "Listing packages..." >&2
-package_dirs=()
-package_names=()
-for dir in $(npx lerna ls -p); do
-    if [[ -f $dir/.jsii ]]; then
-        package_dirs+=("$dir")
-        package_names+=("$(package_name $dir)")
-    fi
-done
+list_jsii_packages() {
+    echo "Listing jsii packages..." >&2
+    for dir in $(npx lerna ls -p); do
+        if [[ -f $dir/.jsii ]]; then
+            echo "$dir"
+        fi
+    done
+}
+jsii_package_dirs=$(list_jsii_packages)
 
 #----------------------------------------------------------------------
 
+# Input a directory, output the package name IF it exists on GitHub
+dirs_to_existing_names() {
+    local dir="$1"
+    local name=$(package_name "$dir")
+    if package_exists_on_npm $name; then
+        echo "$name"
+        echo -n "." >&2
+    else
+        echo -n "x" >&2
+    fi
+}
+
+export -f package_name
+export -f package_exists_on_npm
+export -f dirs_to_existing_names
+
+
 if ! ${SKIP_DOWNLOAD:-false}; then
     echo "Filtering on existing packages on NPM..." >&2
-    for i in ${!package_dirs[@]}; do
-        echo -n "${package_names[$i]}... "
-        if npm view --loglevel silent ${package_names[$i]} > /dev/null; then
-            echo "Exists."
-        else
-            echo "NEW."
-            unset 'package_names[i]'
-            unset 'package_dirs[i]'
-        fi
-    done
+    # In parallel
+    existing_names=$(echo "$jsii_package_dirs" | xargs -n1 -P4 -I {} bash -c 'dirs_to_existing_names "$@"' _ {})
+    echo " Done." >&2
 
     rm -rf $tmpdir
     mkdir -p $tmpdir
 
     echo "Installing from NPM..." >&2
-    (cd $tmpdir && npm install --prefix $tmpdir ${package_names[@]})
+    (cd $tmpdir && npm install --prefix $tmpdir $existing_names)
 fi
 
 #----------------------------------------------------------------------
@@ -50,14 +68,15 @@ current_version=$(npx lerna ls -pl | head -n 1 | cut -d ':' -f 3)
 
 echo "Checking compatibility..." >&2
 success=true
-for i in ${!package_dirs[*]}; do
-    if [[ ! -d $tmpdir/node_modules/${package_names[$i]} ]]; then continue; fi
-    echo -n "${package_names[$i]}... "
+for dir in $jsii_package_dirs; do
+    name=$(package_name "$dir")
+    if [[ ! -d $tmpdir/node_modules/$name ]]; then continue; fi
+    echo -n "$name... "
     if npx jsii-diff \
         --keys \
         --ignore-file ${repo_root}/allowed-breaking-changes.txt \
-        $tmpdir/node_modules/${package_names[$i]} \
-        ${package_dirs[$i]} \
+        $tmpdir/node_modules/$name \
+        $dir \
         2>$tmpdir/output.txt; then
         echo "OK."
     else

scripts/list-packages

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+/**
+ * Collects all packages in the repository in a way that makes it
+ * easy to process for packaging.
+ */
+const child_process = require('child_process');
+const fs = require('fs-extra');
+const path = require('path');
+
+if (process.argv.length < 4) {
+  process.stderr.write('Usage: list-packages <jsii file> <nonjsii file>\n');
+  process.exit(1);
+}
+
+child_process.exec('lerna ls --json', { shell: true }, (error, stdout) => {
+  if (error) {
+    console.error('Error: ', error);
+    process.exit(-1);
+  }
+  const modules = JSON.parse(stdout.toString('utf8'));
+
+  const jsiiDirectories = [];
+  const nonJsiiNames = [];
+
+  for (const module of modules) {
+    const pkgJson = require(path.join(module.location, 'package.json'));
+    if (pkgJson.jsii) {
+      jsiiDirectories.push(module.location);
+    } else {
+      nonJsiiNames.push(pkgJson.name);
+    }
+  }
+
+  fs.writeFileSync(process.argv[2], jsiiDirectories.join('\n'));
+  fs.writeFileSync(process.argv[3], nonJsiiNames.join('\n'));
+});