Merge pull request #6471 from rjbou/gha-depexts

GHA depexts: some enhancements

Author: kit-ty-kate

.github/scripts/depexts/generate-actions.sh

@@ -6,10 +6,10 @@ set -eu
 target=$1
 dir=.github/actions/$target
 
-mkdir -p $dir
+mkdir -p "$dir"
 
 ### Generate the action
-cat >$dir/action.yml << EOF
+cat > "$dir/action.yml" << EOF
 name: 'depexts-$target'
 description: 'Test external dependencies handling for $target'
 runs:
@@ -25,23 +25,23 @@ OCAML_CONSTRAINT=''
 
 case "$target" in
   alpine)
-    cat >$dir/Dockerfile << EOF
+    cat > "$dir/Dockerfile" << EOF
 FROM alpine
 RUN apk add $mainlibs $ocaml
 RUN apk add g++
 EOF
     ;;
   archlinux)
 # no automake
-    cat >$dir/Dockerfile << EOF
+    cat > "$dir/Dockerfile" << EOF
 FROM archlinux
 RUN pacman -Syu --noconfirm $mainlibs $ocaml gcc diffutils
 EOF
     ;;
  centos)
    # CentOS 7 doesn't support OCaml 5 (GCC is too old)
    OCAML_CONSTRAINT=' & < "5.0"'
-    cat >$dir/Dockerfile << EOF
+    cat > "$dir/Dockerfile" << EOF
 FROM almalinux:9.4
 RUN dnf install 'dnf-command(config-manager)' -y
 RUN dnf config-manager --set-enabled crb
@@ -51,15 +51,15 @@ RUN sed -i 's/ID="almalinux"/ID="centos"/' /etc/os-release
 EOF
     ;;
   debian)
-  cat >$dir/Dockerfile << EOF
+  cat > "$dir/Dockerfile" << EOF
 FROM debian
 RUN apt update
 RUN apt install -y $mainlibs $ocaml
 RUN apt install -y g++
 EOF
     ;;
   fedora)
-  cat >$dir/Dockerfile << EOF
+  cat > "$dir/Dockerfile" << EOF
 FROM fedora
 RUN dnf install -y $mainlibs $ocaml diffutils
 RUN dnf install -y gcc-c++
@@ -69,7 +69,7 @@ EOF
   mainlibs=${mainlibs/git/dev-vcs\/git}
   mainlibs=${mainlibs/tar/app-arch\/tar}
   mainlibs=${mainlibs/bzip2/app-arch\/bzip2}
-  cat >$dir/Dockerfile << EOF
+  cat > "$dir/Dockerfile" << EOF
 # name the portage image
 FROM gentoo/portage as portage
 # image is based on stage3
@@ -83,21 +83,21 @@ EOF
     ;;
   opensuse)
   # glpk-dev is installed manually because os-family doesn't handle tumbleweed
-    cat >$dir/Dockerfile << EOF
+    cat > "$dir/Dockerfile" << EOF
 FROM opensuse/leap:15.3
 RUN zypper --non-interactive install $mainlibs $ocaml diffutils gzip glpk-devel
 RUN zypper --non-interactive install gcc-c++
 EOF
     ;;
   oraclelinux)
-    cat >$dir/Dockerfile << EOF
+    cat > "$dir/Dockerfile" << EOF
 FROM oraclelinux:8
 RUN yum install -y $mainlibs
 RUN yum install -y gcc-c++
 EOF
   ;;
   ubuntu)
-  cat >$dir/Dockerfile << EOF
+  cat > "$dir/Dockerfile" << EOF
 FROM ubuntu:20.04
 RUN apt update
 RUN apt install -y $mainlibs $ocaml
@@ -108,10 +108,13 @@ esac
 
 OCAML_INVARIANT="\"ocaml\" {>= \"4.09.0\"$OCAML_CONSTRAINT}"
 
-# Copy 2.1 opam binary from cache
-cp binary/opam $dir/opam
+# Copy released opam binary from cache
+cp binary/opam "$dir/opam"
 
-cat >>$dir/Dockerfile << EOF
+LOCAL_REPO=/opam/repo
+CONF_BRANCH=confs
+
+cat >> "$dir/Dockerfile" << EOF
 RUN test -d /opam || mkdir /opam
 ENV OPAMROOTISOK=1
 ENV OPAMROOT=/opam/root
@@ -120,7 +123,19 @@ ENV OPAMCONFIRMLEVEL=unsafe-yes
 ENV OPAMPRECISETRACKING=1
 COPY opam /usr/bin/opam
 RUN echo 'default-invariant: [ $OCAML_INVARIANT ]' > /opam/opamrc
-RUN /usr/bin/opam init --no-setup --disable-sandboxing --bare --config /opam/opamrc git+$OPAM_REPO#$OPAM_REPO_SHA
+# Retrieve opam repo
+RUN git clone $OPAM_REPO --single-branch --branch master $LOCAL_REPO
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "OPAM GHA"
+RUN git -C $LOCAL_REPO reset --hard $OPAM_REPO_SHA
+RUN git -C $LOCAL_REPO reset --soft \$(git -C $LOCAL_REPO rev-list --all | tail -1)
+RUN git -C $LOCAL_REPO commit -qm "all packages"
+# Build a branch that contains only confs packages
+RUN git -C $LOCAL_REPO checkout -b $CONF_BRANCH
+RUN git -C $LOCAL_REPO rm -q \$(git -C $LOCAL_REPO ls-files packages | grep -v "^packages/conf-")
+RUN git -C $LOCAL_REPO commit -qm "keep only confs"
+# Setup opam
+RUN /usr/bin/opam init --no-setup --disable-sandboxing --bare --config /opam/opamrc git+file://$LOCAL_REPO#master
 RUN echo 'archive-mirrors: "https://opam.ocaml.org/cache"' >> \$OPAMROOT/config
 RUN /usr/bin/opam switch create this-opam --formula='$OCAML_INVARIANT'
 RUN /usr/bin/opam install opam-core opam-state opam-solver opam-repository opam-format opam-client --deps
@@ -131,67 +146,72 @@ EOF
 
 
 ### Generate the entrypoint
-cat >$dir/entrypoint.sh << EOF
+cat > "$dir/entrypoint.sh" << EOF
 #!/bin/sh
 set -eux
 
 git config --global --add safe.directory /github/workspace
 
+## CI WORKING DIR
 # Workdir is /github/workpaces
 cd /github/workspace
 
-### LOCAL TESTING
-#git clone https://github.com/ocaml/opam --single-branch --branch 2.2 --depth 1 local-opam
+## LOCAL TESTING WORKING DIR
+# with docker run -v local/path/opam:/opam/local-git:ro
+#git clone /opam/local-git --single-branch --branch branch-name --depth 1 local-opam
+# with a distant branch
+#git clone https://github.com/ocaml/opam --single-branch --branch branch-name --depth 1 local-opam
 #cd local-opam
 
-opam install . --deps
-eval \$(opam env)
+/usr/bin/opam install . --deps
+eval \$(/usr/bin/opam env)
 ./configure
 make
+
 ./opam config report
-./opam switch create confs --empty
+./opam switch create confs --empty --repo rconf=git+file://$LOCAL_REPO#$CONF_BRANCH
 EOF
 
 # Test depexts
 
 DEPEXTS2TEST=""
 test_depext () {
-  DEPEXTS2TEST="$DEPEXTS2TEST $@"
+  DEPEXTS2TEST="$DEPEXTS2TEST $*"
 }
 
 test_depext conf-gmp.4 conf-which.1
 
-if [ $target != "gentoo" ]; then
+if [ "$target" != gentoo ]; then
   test_depext conf-autoconf.0.1
 fi
 
 # disable automake for centos, as os-family returns rhel
-if [ $target != "centos" ] && [ $target != "gentoo" ] && [ $target != "opensuse" ]; then
+if [ "$target" != centos ] && [ "$target" != gentoo ] && [ "$target" != opensuse ]; then
   test_depext conf-automake.1
 fi
 
-# additionna
-if [ $target != "oraclelinux" ] && [ $target != "xxx" ]; then
+# additional
+if [ "$target" != oraclelinux ]; then
   test_depext conf-dpkg.1 # gentoo
 fi
 
 # package with os-version check
 
-if [ $target = "debian" ] || [ $target = "ubuntu" ]; then
+if [ "$target" = debian ] || [ "$target" = ubuntu ]; then
   test_depext conf-sundials.2
   # conf-libgccjit.1 conf-rdkit.1
 fi
 
-if [ $target = "alpine" ]; then
+if [ "$target" = alpine ]; then
  test_depext conf-clang-format.1
  # conf-pandoc.0.1
 fi
 
-if [ $target = "fedora" ]; then
+if [ "$target" = fedora ]; then
  test_depext conf-emacs.1
 fi
 
-if [ $target = "oraclelinux" ] || [ $target = "centos" ]; then
+if [ "$target" = oraclelinux ] || [ "$target" = centos ]; then
   test_depext conf-pkg-config.3
 fi
 
@@ -203,7 +223,7 @@ if [ -z "$DEPEXTS2TEST" ]; then
   exit 3
 fi
 
-cat >>$dir/entrypoint.sh << EOF
+cat >> "$dir/entrypoint.sh" << EOF
 ERRORS=""
 test_depexts () {
   for pkg in \$@ ; do
@@ -222,10 +242,10 @@ fi
 EOF
 
 # Test depexts update
-cat >>$dir/entrypoint.sh << EOF
+cat >> "$dir/entrypoint.sh" << EOF
 ./opam update --depexts || ERRORS="\$ERRORS opam-update-depexts"
 EOF
 
-chmod +x $dir/entrypoint.sh
+chmod +x "$dir/entrypoint.sh"
 
 #done

.github/scripts/main/hygiene.sh

@@ -135,6 +135,12 @@ else
   (set +x; echo -e "shell/install.sh: \e[31mERROR\e[0m") 2>/dev/null
   ERROR=1
 fi
+if shellcheck .github/scripts/depexts/generate-actions.sh ; then
+  (set +x; echo ".github/scripts/depexts/generate-actions.sh: OK") 2>/dev/null
+else
+  (set +x; echo -e ".github/scripts/depexts/generate-actions.sh: \e[31mERROR\e[0m") 2>/dev/null
+  ERROR=1
+fi
 (set +x ; echo -en "::endgroup::check shell scripts using shellcheck\r") 2>/dev/null
 
 exit $ERROR

master_changes.md

@@ -231,6 +231,11 @@ users)
   * Show Cygwin version info after loading it from the cache [#6383 @kit-ty-kate]
   * Upgrade the opam version used to setup the depexts workflows to 2.3.0 [#6444 @kit-ty-kate]
   * Enable the depexts workflow on master and when the opam description change [#6447 @kit-ty-kate]
+  * Make depext generate action script pass spellcheck [#6471 @kit-ty-kate]
+  * Add depext generate action script to spellcheck check action [#6471 @kit-ty-kate]
+  * Some cleaning in depext generate action and instruction for local testing [#6471 @rjbou]
+  * Cache the repository and a minimal one in docker image for depext jobs [#6471 @rjbou]
+  * Use the good opam binary in depext jobs [#6471 @RyanGibb]
 
 ## Doc
   * Update the command to install opam to point to the new simplified url on opam.ocaml.org [#6226 @kit-ty-kate]