Merge pull request #4826 from kinke/gha_musl

GHA: Integrate Alpine musl job in main workflow

Author: kinke

.github/actions/4c-test-dmd/action.yml

@@ -9,7 +9,20 @@ runs:
     - name: 'Posix: Run DMD testsuite'
       if: runner.os != 'Windows'
       shell: bash
-      run: cd ../build && ctest -V -R "dmd-testsuite"
+      run: |
+        set -eux
+        repoDir=$PWD
+        cd ../build
+        if type -P apk &>/dev/null; then
+          # Alpine: run full dmd-testsuite-debug
+          ctest -V -R 'dmd-testsuite' -E '^dmd-testsuite$'
+          # these two tests require extra flags "-link-defaultlib-debug -frame-pointer=all": https://github.com/ldc-developers/ldc/issues/4694
+          # => remove before running optimized dmd-testsuite separately
+          rm $repoDir/tests/dmd/runnable/{test17559.d,test19086.d}
+          ctest -V -R '^dmd-testsuite$'
+        else
+          ctest -V -R "dmd-testsuite"
+        fi
 
     - name: 'Windows: Run DMD testsuite'
       if: runner.os == 'Windows'

.github/actions/6-integration-test/action.yml

@@ -21,7 +21,7 @@ runs:
         fi
         installed/bin/ldc2 hello.d -link-defaultlib-shared
         ./hello
-        if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then
+        if [[ -d installed/lib32 ]]; then
           installed/bin/ldc2 hello.d -m32 -link-defaultlib-shared
           ./hello
         fi
@@ -34,7 +34,7 @@ runs:
         for mode in thin full; do
           installed/bin/ldc2 hello.d -of=hello_$mode -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
           ./hello_$mode
-          if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then
+          if [[ -d installed/lib32 ]]; then
             installed/bin/ldc2 hello.d -m32 -of=hello_$mode-32 -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
             ./hello_$mode-32
           fi
@@ -45,6 +45,9 @@ runs:
       run: |
         set -eux
         cd ..
+        if type -P apk &>/dev/null; then
+          exit 0 # Alpine: disabled dynamic-compile support
+        fi
         if [[ '${{ runner.os }}' == Windows ]]; then
           # add ldc-jit.dll dir to PATH
           export PATH="$PWD/installed/bin:$PATH"

.github/actions/7-package/action.yml

@@ -49,6 +49,11 @@ runs:
         cd dlang-tools
         git checkout "$(cat ../ldc/packaging/dlang-tools_version)"
 
+        # Alpine: build these tools as fully static executables
+        if [[ '${{ inputs.os }}' == 'alpine' ]]; then
+          export DFLAGS="-Xcc=-static${DFLAGS:+ $DFLAGS}"
+        fi
+
         mkdir bin
         $DMD -w -de -dip1000 rdmd.d -of=bin/rdmd
         $DMD -w -de -dip1000 ddemangle.d -of=bin/ddemangle
@@ -73,16 +78,29 @@ runs:
           "$(dirname "$DMD")/ldc2" -run list_payload.d .
         fi
 
-        # use host compiler's dub, which is guaranteed to be native
-        DFLAGS="-O -linkonce-templates ${DFLAGS:-}" dub build -v \
-          --build-mode=allAtOnce --combined $archFlag \
-          --compiler="$(dirname "$DMD")/ldc2"
+        # prefer host compiler's dub, which is guaranteed to be native
+        PATH="$PATH:$PWD/../installed/bin" \
+          DFLAGS="-O -linkonce-templates ${DFLAGS:-}" \
+          dub build -v \
+            --build-mode=allAtOnce --combined $archFlag \
+            --compiler="$(dirname "$DMD")/ldc2"
         cp bin/reggae ../installed/bin/
 
         if [[ '${{ inputs.cross_target_triple }}' == '' ]]; then
           ../installed/bin/reggae --version -b ninja
         fi
 
+    - name: 'Linux: List executable dependencies'
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -euxo pipefail
+        cd ../installed/bin
+        for i in *; do
+          ls -lh $i
+          readelf -d $i | grep NEEDED || true
+        done
+
     - name: Pack installation dir
       shell: bash
       run: |
@@ -120,10 +138,11 @@ runs:
           chmod -R go=rX $artifactName
           if [[ '${{ runner.os }}' == macOS ]]; then
             sudo chown -R root:wheel $artifactName
-            tar -cf - $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
-          else
-            tar -cf - --owner=0 --group=0 $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
+          # note: already running as root in Alpine container, *and* potential busybox complication
+          elif [[ $os != alpine ]]; then
+            sudo chown -R 0:0 $artifactName
           fi
+          tar -cf - $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
         fi
 
         # export ARTIFACT_{ID,NAME}

.github/workflows/alpine_musl.yml

@@ -53,6 +53,27 @@ jobs:
               -DEXTRA_CXXFLAGS=-flto=full
             with_pgo: true
 
+          - job_name: Alpine musl x86_64
+            os: ubuntu-latest
+            container_image: alpine:3.20
+            arch: x86_64
+            bootstrap_cmake_flags: -DBUILD_LTO_LIBS=ON
+            # TSan and XRay do not work.
+            extra_cmake_flags: >-
+              -DBUILD_LTO_LIBS=ON
+              -DLLVM_IS_SHARED=OFF
+              -DLDC_ENABLE_PLUGINS=OFF
+              -DLDC_DYNAMIC_COMPILE=OFF
+              -DLDC_INSTALL_LTOPLUGIN=OFF
+              -DCOMPILER_RT_LIBDIR_OS=linux
+              -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=x86_64
+              -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer"
+              -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++
+              -DLDC_FULLY_STATIC=ON
+              -DD_COMPILER_FLAGS="-O -flto=full"
+            # FIXME: -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
+            with_pgo: false
+
           - job_name: macOS x86_64
             os: macos-13
             arch: x86_64
@@ -108,18 +129,33 @@ jobs:
     env:
       MACOSX_DEPLOYMENT_TARGET: ${{ matrix.arch == 'arm64' && '11.0' || '10.12' }}
     steps:
-      - name: 'Container: Install git and sudo'
+      - name: 'Set up Linux container'
         if: matrix.container_image
-        shell: bash
         run: |
           set -eux
-          apt-get -q update
-          DEBIAN_FRONTEND=noninteractive apt-get -yq install git-core sudo
+          if type -P apt-get &>/dev/null; then
+            # Ubuntu: pre-install git and sudo
+            apt-get -q update
+            DEBIAN_FRONTEND=noninteractive apt-get -yq install git-core sudo
+          else
+            # set up Alpine container
+            apk add \
+              git cmake ninja-is-really-ninja g++ ldc llvm-dev llvm-static compiler-rt \
+              libxml2-static zstd-static zlib-static \
+              bash grep diffutils make curl 7zip perl
+            # create missing 7za symlink
+            ln -s 7z /usr/bin/7za
+            # create ../llvm symlink to distro LLVM (no prebuilt LDC-LLVM for musl)
+            ln -s /usr/lib/llvm17 $(dirname $(pwd))/llvm
+            ../llvm/bin/llvm-config --version
+          fi
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 50
       - name: Install prerequisites
+        if: |
+          !startsWith(matrix.container_image, 'alpine')
         uses: ./.github/actions/1-setup
         with:
           llvm_version: ${{ matrix.llvm_version || env.LLVM_VERSION }}
@@ -194,6 +230,7 @@ jobs:
         uses: ./.github/actions/7-package
         with:
           arch: ${{ matrix.arch }}
+          os: ${{ startsWith(matrix.container_image, 'alpine') && 'alpine' || '' }}
 
 
   # Cross-compilation jobs for non-native targets.

.github/workflows/main.yml

@@ -2,6 +2,7 @@
 
 #### Big news
 - LLVM for prebuilt packages bumped to v19.1.7. (#4822)
+- New prebuilt package for Alpine Linux x86_64 with musl libc. It's currently generated on Alpine v3.20, using its default LLVM 17. Most bundled executables are fully static and can be run on ~all distros. (#4826)
 - Revived dynamic-compile (JIT) functionality (formerly unsupported since LLVM 12), supporting LLVM 18+ now. (#4774)
 - ldc2.conf: `%%ldcversion%%` placeholder added, allowing to refer to version-specific directories.
 

CHANGELOG.md

@@ -315,6 +315,12 @@ else()
     separate_arguments(LLVM_LDFLAGS UNIX_COMMAND "${LLVM_LDFLAGS}")
 endif()
 
+option(LDC_FULLY_STATIC "Posix: Link most executables (compiler and tools) as fully-static binaries" OFF)
+set(FULLY_STATIC_LDFLAG)
+if(UNIX AND LDC_FULLY_STATIC)
+    set(FULLY_STATIC_LDFLAG "-static")
+endif()
+
 # Suppress superfluous randlib warnings about "*.a" having no symbols on MacOSX.
 if (APPLE)
     set(CMAKE_C_ARCHIVE_CREATE   "<CMAKE_AR> Scr <TARGET> <LINK_FLAGS> <OBJECTS>")
@@ -708,7 +714,7 @@ build_d_executable(
     "${LDC_EXE_FULL}"
     "${LDC_D_SOURCE_FILES}"
     "${DFLAGS_BUILD_TYPE} ${DFLAGS_LDC}"
-    "${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST}"
+    "${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}"
     "${FE_RES}"
     "${LDC_LIB}"
     ${COMPILE_D_MODULES_SEPARATELY}
@@ -743,7 +749,7 @@ build_d_executable(
     "${LDMD_EXE_FULL}"
     "${LDMD_D_SOURCE_FILES}"
     "${DFLAGS_BUILD_TYPE}"
-    "${LDC_LINKERFLAG_LIST}"
+    "${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}"
     ""
     "LDMD_CXX_LIB"
     ${COMPILE_D_MODULES_SEPARATELY}
@@ -927,7 +933,7 @@ build_d_executable(
     "${LDC_UNITTEST_EXE_FULL}"
     "${LDC_D_SOURCE_FILES}"
     "-g -unittest ${DFLAGS_LDC}"
-    "${LDC_LINKERFLAG_LIST}"
+    "${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}"
     ""
     "${LDC_LIB}"
     ${COMPILE_D_MODULES_SEPARATELY}
@@ -957,7 +963,7 @@ build_d_executable(
     "${LDC_BUILD_RUNTIME_EXE_FULL}"
     "${PROJECT_BINARY_DIR}/ldc-build-runtime.d"
     "${DFLAGS_BUILD_TYPE}"
-    ""
+    "" # must not be fully static for std.net.curl support (loading libcurl dynamically)
     "${PROJECT_SOURCE_DIR}/runtime/ldc-build-runtime.d.in"
     ""
     ${COMPILE_D_MODULES_SEPARATELY}

CMakeLists.txt

@@ -95,7 +95,7 @@ function(build_d_executable target_name output_exe d_src_files compiler_args lin
             list(APPEND dep_libs "-L$<TARGET_LINKER_FILE:${l}>")
         endforeach()
 
-        set(full_linker_args ${CMAKE_EXE_LINKER_FLAGS} ${linker_args})
+        set(full_linker_args ${CMAKE_EXE_LINKER_FLAGS} ${linker_args} ${D_LINKER_ARGS})
         translate_linker_args(full_linker_args translated_linker_args)
 
         # We need to link against the C++ runtime library.

cmake/Modules/BuildDExecutable.cmake

@@ -80,9 +80,16 @@ else()
        if(LLVM_FIND_QUIETLY)
             set(_quiet_arg ERROR_QUIET)
         endif()
+        if(DEFINED LLVM_IS_SHARED)
+            if(LLVM_IS_SHARED)
+                set(_sharedstatic "--link-shared")
+            else()
+                set(_sharedstatic "--link-static")
+            endif()
+        endif()
         set(result_code)
         execute_process(
-            COMMAND ${LLVM_CONFIG} --${flag}
+            COMMAND ${LLVM_CONFIG} ${_sharedstatic} --${flag}
             RESULT_VARIABLE result_code
             OUTPUT_VARIABLE LLVM_${var}
             OUTPUT_STRIP_TRAILING_WHITESPACE
@@ -100,9 +107,14 @@ else()
        if(LLVM_FIND_QUIETLY)
             set(_quiet_arg ERROR_QUIET)
         endif()
+        if (LLVM_IS_SHARED)
+            set(_sharedstatic "--link-shared")
+        else()
+            set(_sharedstatic "--link-static")
+        endif()
         set(result_code)
         execute_process(
-            COMMAND ${LLVM_CONFIG} --${flag} ${components}
+            COMMAND ${LLVM_CONFIG} ${_sharedstatic} --${flag} ${components}
             RESULT_VARIABLE result_code
             OUTPUT_VARIABLE tmplibs
             OUTPUT_STRIP_TRAILING_WHITESPACE
@@ -116,6 +128,15 @@ else()
         endif()
     endmacro()
 
+    if (NOT DEFINED LLVM_IS_SHARED)
+        llvm_set(SHARED_MODE shared-mode)
+        if(LLVM_SHARED_MODE STREQUAL "shared")
+            set(LLVM_IS_SHARED ON)
+        else()
+            set(LLVM_IS_SHARED OFF)
+        endif()
+    endif()
+
     llvm_set(VERSION_STRING version)
     llvm_set(CXXFLAGS cxxflags)
     llvm_set(INCLUDE_DIRS includedir true)
@@ -127,13 +148,6 @@ else()
     string(REGEX REPLACE "([0-9]+).*" "\\1" LLVM_VERSION_MAJOR "${LLVM_VERSION_STRING}" )
     string(REGEX REPLACE "[0-9]+\\.([0-9]+).*[A-Za-z]*" "\\1" LLVM_VERSION_MINOR "${LLVM_VERSION_STRING}" )
 
-    llvm_set(SHARED_MODE shared-mode)
-    if(LLVM_SHARED_MODE STREQUAL "shared")
-        set(LLVM_IS_SHARED ON)
-    else()
-        set(LLVM_IS_SHARED OFF)
-    endif()
-
     llvm_set(LDFLAGS ldflags)
     llvm_set(SYSTEM_LIBS system-libs)
     string(REPLACE "\n" " " LLVM_LDFLAGS "${LLVM_LDFLAGS} ${LLVM_SYSTEM_LIBS}")