Merge pull request #185 from freedomofpress/182-build-tarballs-dynamically

Builds source tarballs dynamically

Author: kushaldas

scripts/build-debianpackage

@@ -27,29 +27,93 @@ TOP_BUILDDIR="$HOME/debbuild/packaging"
 mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
+# Move changelog into place (we have separate changelogs for each platform)
+PLATFORM="$(lsb_release -sc)"
 
 # Validate required args.
 if [[ -z "${PKG_NAME:-}" ]]; then
     echo "Set PKG_NAME of the build";
     exit 1
 fi
 
+
+# Look up most recent release from GitHub repo
+function find_latest_version() {
+    repo_url="https://github.com/freedomofpress/${PKG_NAME}/releases"
+    curl -s "$repo_url" \
+        | perl -nE '$_ =~ m#/releases/tag/(v?[\d\.]+)\"# and say $1' \
+        | head -n 1
+}
+
 if [[ -z "${PKG_VERSION:-}" ]]; then
-    echo "Set PKG_VERSION of the build";
-    exit 1
+    echo "PKG_VERSION not set, inferring from recent releases..."
+    PKG_VERSION="$(find_latest_version)"
+    if [[ -z "$PKG_VERSION" ]]; then
+        echo "Failed to infer version"
+        exit 1
+    else
+        echo "Using PKG_VERSION: $PKG_VERSION"
+    fi
 fi
 
 # Copy over the debian directory (including new changelog) from repo
 cp -r "$CUR_DIR/$PKG_NAME/" "$TOP_BUILDDIR/"
 
+# Ensures that a given git tag is signed with the prod release key
+# If "rc" is in the tag name, this will fail.
+function verify_git_tag() {
+    local d
+    local t
+    d="$1"
+    t="$2"
+    prod_fingerprint="22245C81E3BAEB4138B36061310F561200F4AD77"
+    if ! git -C "$build_dir" tag --verify "$PKG_VERSION" 2>&1 \
+        | grep -q -F "using RSA key $prod_fingerprint" ; then
+        echo "Failed to verify $PKG_VERSION, not signed with $prod_fingerprint" >&2
+        exit 2
+    fi
+}
+
+# Dynamically generate a tarball, from the Python source code,
+# that is byte-for-byte reproducible. Infers timestamp
+# from the changelog, same as for the deb package.
+function build_source_tarball() {
+    repo_url="https://github.com/freedomofpress/${PKG_NAME}"
+    build_dir="/tmp/${PKG_NAME}"
+    rm -rf "$build_dir"
+    git clone "$repo_url" "$build_dir"
+
+    # Verify tag, using only the prod key
+    verify_git_tag "$build_dir" "$PKG_VERSION"
+
+    # Tag is verified, proceed with checkout
+    git -C "$build_dir" checkout "$PKG_VERSION"
+    (cd "$build_dir" && LC_ALL="C.UTF-8" python setup.py sdist)
+
+    # Initial tarball will contain timestamps from NOW, let's repack
+    # with timestamps from the changelog, which is static.
+    raw_tarball="$(find "${build_dir}/dist/" | grep -P '\.tar.gz$' | head -n1)"
+    dch_time="$(date "+%Y-%m-%d %H:%M:%S %z" -d@$(dpkg-parsechangelog --file $PKG_NAME/debian/changelog-$PLATFORM -STimestamp)) "
+    (cd "$build_dir" && tar -xzf "dist/$(basename $raw_tarball)")
+    tarball_basename="$(basename "$raw_tarball")"
+    # Repack with tar only, so env vars are respected
+    (cd "$build_dir" && tar -cf "${tarball_basename%.gz}" --mode=go=rX,u+rw,a-s --mtime="$dch_time" --sort=name --owner=root:0 --group=root:0 "${tarball_basename%.tar.gz}" 1>&2)
+    # Then gzip it separately, so we can pass args
+    (cd "$build_dir" && gzip --no-name "${tarball_basename%.gz}")
+    (cd "$build_dir" && mv "$tarball_basename" dist/)
+    echo "$raw_tarball"
+}
+
 # If the package is contained in the list, it should be a python package. In
 # that case, we should extract tarball, and validate wheel hashes.
 if [[ "${PKG_NAME}" =~ ^(securedrop-client|securedrop-proxy|securedrop-export|securedrop-log)$ ]]; then
     echo "${PKG_NAME} is a Python package"
 
     if [[ -z "${PKG_PATH:-}" ]]; then
-        # Try to find tarball in a reasonable location
-        candidate_pkg_path="$(realpath "${CUR_DIR}/../${PKG_NAME}/dist/${PKG_NAME}-${PKG_VERSION}.tar.gz")"
+        # Build from source
+        echo "PKG_PATH not set, building from source (version $PKG_VERSION)..."
+        build_source_tarball
+        candidate_pkg_path="$(find /tmp/$PKG_NAME/dist -type f -iname '*.tar.gz')"
         if [[ -f "$candidate_pkg_path" ]]; then
             PKG_PATH="$candidate_pkg_path"
             echo "Found tarball at $PKG_PATH, override with PKG_PATH..."
@@ -80,8 +144,6 @@ fi
 
 printf "Building package '%s' from version '%s'...\\n" "$PKG_NAME" "$PKG_VERSION"
 
-# Move changelog into place (we have separate changelogs for each platform)
-PLATFORM="$(lsb_release -sc)"
 echo "$TOP_BUILDDIR/$PKG_NAME/"
 mv "$TOP_BUILDDIR/$PKG_NAME/debian/changelog-$PLATFORM" "$TOP_BUILDDIR/$PKG_NAME/debian/changelog"