awslabs · hughcars · May 21, 2025 · May 12, 2025 · May 13, 2025 · May 13, 2025
diff --git a/.github/workflows/singularity.yml b/.github/workflows/singularity.yml
@@ -6,6 +6,7 @@ on:
       - main
     tags: '*'
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -86,6 +86,9 @@ The format of this changelog is based on
   - Added `PortExcitations` to manage excitation pattern and print it to json metadata.
   - Update spack configuration to address `0.12` API breakage between libxsmm and libCEED.
   - Support spack vended libCEED and GSLIB builds.
+  - Fix bug where `MakeSimplex` would fail for higher order meshes.
+  - Fix bug in STRUMPACK build where dependency SCALAPACK library install directory could
+    change based on build environment.
 
 ## [0.13.0] - 2024-05-20
 

diff --git a/cmake/ExternalGitTags.cmake b/cmake/ExternalGitTags.cmake
@@ -34,7 +34,7 @@ set(EXTERN_BUTTERFLYPACK_GIT_BRANCH
   "Git branch for external ButterflyPACK build"
 )
 set(EXTERN_BUTTERFLYPACK_GIT_TAG
-  "3ea057d9c00b319c854361e8d38b18f80b35d09c" CACHE STRING
+  "cbbee2999ea8ec82a1f9d06320522db22d5d3ad2" CACHE STRING
   "Git tag for external ButterflyPACK build"
 )
 
@@ -62,7 +62,7 @@ set(EXTERN_HYPRE_GIT_BRANCH
   "Git branch for external HYPRE build"
 )
 set(EXTERN_HYPRE_GIT_TAG
-  "19ecc97885938dabc20d049759aad0309fa5ff2e" CACHE STRING
+  "4f6f6784f47976d6e1f2e4750a1d02f7cb56e127" CACHE STRING
   "Git tag for external HYPRE build"
 )
 
@@ -76,7 +76,7 @@ set(EXTERN_LIBCEED_GIT_BRANCH
   "Git branch for external libCEED build"
 )
 set(EXTERN_LIBCEED_GIT_TAG
-  "6b50d2be0e7f583c08aab912c64a7bf4147ef2ad" CACHE STRING
+  "4db02c8f89970d9e4874124af1a6d3e1d3666a88" CACHE STRING
   "Git tag for external libCEED build"
 )
 
@@ -104,7 +104,7 @@ set(EXTERN_MAGMA_GIT_BRANCH
   "Git branch for external MAGMA build"
 )
 set(EXTERN_MAGMA_GIT_TAG
-  "48a32d79517cc6f1522aa190cdec4b1d1babcf53" CACHE STRING
+  "4fc6d1fc831ec73568f665ba7a3468fa82ec6f0c" CACHE STRING
   "Git tag for external MAGMA build"
 )
 
@@ -118,7 +118,7 @@ set(EXTERN_METIS_GIT_BRANCH
   "Git branch for external METIS build"
 )
 set(EXTERN_METIS_GIT_TAG
-  "69fb26dd042836aa16f26fb939b540c5ca71133d" CACHE STRING
+  "08c3082720ff9114b8e3cbaa4484a26739cd7d2d" CACHE STRING
   "Git tag for external METIS build"
 )
 
@@ -132,7 +132,7 @@ set(EXTERN_MFEM_GIT_BRANCH
   "Git branch for external MFEM build"
 )
 set(EXTERN_MFEM_GIT_TAG
-  "c87ded944ff903f7ebcb0511c86883a989a18bb0" CACHE STRING
+  "9f945a5863407bdb5e1212465687ba09a49bbd7e" CACHE STRING
   "Git tag for external MFEM build"
 )
 
@@ -146,7 +146,7 @@ set(EXTERN_MUMPS_GIT_BRANCH
   "Git branch for external MUMPS build"
 )
 set(EXTERN_MUMPS_GIT_TAG
-  "87a9700ca0c55052fc90b58a0d05d2df3ef02b10" CACHE STRING
+  "1cfd19699702f9a64ff5d45827d6025ff5c3873a" CACHE STRING
   "Git tag for external MUMPS build"
 )
 
@@ -160,7 +160,7 @@ set(EXTERN_PARMETIS_GIT_BRANCH
   "Git branch for external ParMETIS build"
 )
 set(EXTERN_PARMETIS_GIT_TAG
-  "f5e3aab04fd5fe6e09fa02f885c1c29d349f9f8b" CACHE STRING
+  "53c9341b6c1ba876c97567cb52ddfc87c159dc36" CACHE STRING
   "Git tag for external ParMETIS build"
 )
 
@@ -174,7 +174,7 @@ set(EXTERN_PETSC_GIT_BRANCH
   "Git branch for external PETSc build"
 )
 set(EXTERN_PETSC_GIT_TAG
-  "93392d187e13c9ce8919ac8e80e98f7168db66e1" CACHE STRING
+  "226f8a8a5081bc6ad7227cd631662400f0d6e2a0" CACHE STRING
   "Git tag for external PETSc build"
 )
 
@@ -188,7 +188,7 @@ set(EXTERN_SCALAPACK_GIT_BRANCH
   "Git branch for external ScaLAPACK build"
 )
 set(EXTERN_SCALAPACK_GIT_TAG
-  "a23c2cdc6586c427686f6097ae66bb54ef693571" CACHE STRING
+  "0e8767285b7a201c7b1ff34d2c2bb009534145df" CACHE STRING
   "Git tag for external ScaLAPACK build"
 )
 
@@ -202,7 +202,7 @@ set(EXTERN_SLEPC_GIT_BRANCH
   "Git branch for external SLEPc build"
 )
 set(EXTERN_SLEPC_GIT_TAG
-  "a4e1bb593f369cb8a2f4b846fda09e539b0547b5" CACHE STRING
+  "94cd514ee23a6958ceca39b49c8ef779eb18da6c" CACHE STRING
   "Git tag for external SLEPc build"
 )
 
@@ -216,7 +216,7 @@ set(EXTERN_STRUMPACK_GIT_BRANCH
   "Git branch for external STRUMPACK build"
 )
 set(EXTERN_STRUMPACK_GIT_TAG
-  "115b152be9a5d0d77846e3694f699c53c93fe394" CACHE STRING
+  "f7aeea36325d9dfe60572858599a26575daf15d1" CACHE STRING
   "Git tag for external STRUMPACK build"
 )
 
@@ -230,7 +230,7 @@ set(EXTERN_SUPERLU_GIT_BRANCH
   "Git branch for external SuperLU_DIST build"
 )
 set(EXTERN_SUPERLU_GIT_TAG
-  "d2f0f1e11a4b41c6bb8a1b80554e2339e76c9e8b" CACHE STRING
+  "829615e35d6c4627a941e4ec4f60a7fed21bc2c4" CACHE STRING
   "Git tag for external SuperLU_DIST build"
 )
 
@@ -244,7 +244,7 @@ set(EXTERN_ZFP_GIT_BRANCH
   "Git branch for external ZFP build"
 )
 set(EXTERN_ZFP_GIT_TAG
-  "8fe0059a33e298081409a520c254ebe1836bc20a" CACHE STRING
+  "7d1e3a21047a976599b562b3bbd53b1f34348f1a" CACHE STRING
   "Git tag for external ZFP build"
 )
 
@@ -276,6 +276,6 @@ set(EXTERN_SUNDIALS_GIT_BRANCH
   "Git branch for external SUNDIALS build"
 )
 set(EXTERN_SUNDIALS_GIT_TAG
-  "5c53be85c88f63c5201c130b8cb2c686615cfb03" CACHE STRING
+  "e941546af1a5b5e492dcac0a1872540e9961c556" CACHE STRING
   "Git tag for external SUNDIALS build"
 )
diff --git a/cmake/ExternalMETIS.cmake b/cmake/ExternalMETIS.cmake
@@ -53,11 +53,6 @@ endif()
 string(REPLACE ";" "; " METIS_OPTIONS_PRINT "${METIS_OPTIONS}")
 message(STATUS "METIS_OPTIONS: ${METIS_OPTIONS_PRINT}")
 
-# Some build fixes
-set(METIS_PATCH_FILES
-  "${CMAKE_SOURCE_DIR}/extern/patch/metis/patch_build.diff"
-)
-
 include(ExternalProject)
 ExternalProject_Add(metis
   DEPENDS           ${METIS_DEPENDENCIES}
@@ -68,7 +63,6 @@ ExternalProject_Add(metis
   INSTALL_DIR       ${CMAKE_INSTALL_PREFIX}
   PREFIX            ${CMAKE_BINARY_DIR}/extern/metis-cmake
   UPDATE_COMMAND    ""
-  PATCH_COMMAND     git apply "${METIS_PATCH_FILES}"
   CONFIGURE_COMMAND ${CMAKE_COMMAND} <SOURCE_DIR> ${METIS_OPTIONS}
   TEST_COMMAND      ""
 )
@@ -120,11 +114,6 @@ if(PALACE_WITH_SUPERLU OR PALACE_WITH_STRUMPACK OR PALACE_WITH_MUMPS)
   string(REPLACE ";" "; " PARMETIS_OPTIONS_PRINT "${PARMETIS_OPTIONS}")
   message(STATUS "PARMETIS_OPTIONS: ${PARMETIS_OPTIONS_PRINT}")
 
-  # Some build fixes
-  set(PARMETIS_PATCH_FILES
-    "${CMAKE_SOURCE_DIR}/extern/patch/parmetis/patch_build.diff"
-  )
-
   ExternalProject_Add(parmetis
     DEPENDS           ${PARMETIS_DEPENDENCIES}
     GIT_REPOSITORY    ${EXTERN_PARMETIS_URL}
@@ -134,7 +123,6 @@ if(PALACE_WITH_SUPERLU OR PALACE_WITH_STRUMPACK OR PALACE_WITH_MUMPS)
     INSTALL_DIR       ${CMAKE_INSTALL_PREFIX}
     PREFIX            ${CMAKE_BINARY_DIR}/extern/parmetis-cmake
     UPDATE_COMMAND    ""
-    PATCH_COMMAND     git apply "${PARMETIS_PATCH_FILES}"
     CONFIGURE_COMMAND ${CMAKE_COMMAND} <SOURCE_DIR> "${PARMETIS_OPTIONS}"
     TEST_COMMAND      ""
   )

diff --git a/cmake/ExternalMFEM.cmake b/cmake/ExternalMFEM.cmake
@@ -391,9 +391,9 @@ message(STATUS "MFEM_OPTIONS: ${MFEM_OPTIONS_PRINT}")
 set(MFEM_PATCH_FILES
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_mfem_device_fixes.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_mesh_vis_dev.diff"
-  "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_mesh_prism_vtu_fix.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_par_tet_mesh_fix_dev.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_gmsh_parser_performance.diff"
+  "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_ho_make_simplicial_dev.diff"
 )
 
 include(ExternalProject)

diff --git a/cmake/ExternalScaLAPACK.cmake b/cmake/ExternalScaLAPACK.cmake
@@ -50,12 +50,13 @@ ExternalProject_Add(scalapack
   TEST_COMMAND      ""
 )
 
+include(GNUInstallDirs)
 # Save variables to cache
 if(BUILD_SHARED_LIBS)
   set(_SCALAPACK_LIB_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX})
 else()
   set(_SCALAPACK_LIB_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
 endif()
-set(SCALAPACK_LIBRARIES ${CMAKE_INSTALL_PREFIX}/lib/libscalapack${_SCALAPACK_LIB_SUFFIX}
+set(SCALAPACK_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libscalapack${_SCALAPACK_LIB_SUFFIX}
   CACHE STRING "List of library files for ScaLAPACK"
 )
diff --git a/extern/patch/STRUMPACK/patch_gpu_init.diff b/extern/patch/STRUMPACK/patch_gpu_init.diff
@@ -1,45 +1,53 @@
 diff --git a/src/dense/CUDAWrapper.cpp b/src/dense/CUDAWrapper.cpp
-index 56405d83..50d1f34c 100644
+index 6236a723..532729f5 100644
 --- a/src/dense/CUDAWrapper.cpp
 +++ b/src/dense/CUDAWrapper.cpp
-@@ -306,19 +306,19 @@ namespace strumpack {
+@@ -317,23 +317,23 @@ namespace strumpack {
      }
 
      void init() {
 -#if defined(STRUMPACK_USE_MPI)
 -      int devs;
--      cudaGetDeviceCount(&devs);
+-      gpu_check(cudaGetDeviceCount(&devs));
 -      if (devs > 1) {
 -        int flag, rank = 0;
 -        MPI_Initialized(&flag);
 -        if (flag) {
 -          MPIComm c;
 -          rank = c.rank();
 -        }
--        cudaSetDevice(rank % devs);
+-        gpu_check(cudaSetDevice(rank % devs));
+-#pragma omp parallel
+-        {
+-          gpu_check(cudaSetDevice(rank % devs));
+-        }
 -      }
 -#endif
 +// #if defined(STRUMPACK_USE_MPI)
 +//       int devs;
-+//       cudaGetDeviceCount(&devs);
++//       gpu_check(cudaGetDeviceCount(&devs));
 +//       if (devs > 1) {
 +//         int flag, rank = 0;
 +//         MPI_Initialized(&flag);
 +//         if (flag) {
 +//           MPIComm c;
 +//           rank = c.rank();
 +//         }
-+//         cudaSetDevice(rank % devs);
++//         gpu_check(cudaSetDevice(rank % devs));
++// #pragma omp parallel
++//         {
++//           gpu_check(cudaSetDevice(rank % devs));
++//         }
 +//       }
 +// #endif
        //       gpu_check(cudaFree(0));
        // #if defined(STRUMPACK_USE_MAGMA)
        //       magma_init();
 diff --git a/src/dense/HIPWrapper.cpp b/src/dense/HIPWrapper.cpp
-index b0f6edf3..0025f9f5 100644
+index c68623cf..6fd8f425 100644
 --- a/src/dense/HIPWrapper.cpp
 +++ b/src/dense/HIPWrapper.cpp
-@@ -280,19 +280,19 @@ namespace strumpack {
+@@ -280,23 +280,23 @@ namespace strumpack {
      }
 
      void init() {
@@ -54,6 +62,10 @@ index b0f6edf3..0025f9f5 100644
 -          rank = c.rank();
 -        }
 -        gpu_check(hipSetDevice(rank % devs));
+-#pragma omp parallel
+-        {
+-          gpu_check(hipSetDevice(rank % devs));
+-        }
 -      }
 -#endif
 +// #if defined(STRUMPACK_USE_MPI)
@@ -67,8 +79,21 @@ index b0f6edf3..0025f9f5 100644
 +//           rank = c.rank();
 +//         }
 +//         gpu_check(hipSetDevice(rank % devs));
++// #pragma omp parallel
++//         {
++//           gpu_check(hipSetDevice(rank % devs));
++//         }
 +//       }
 +// #endif
      }
 
      void device_memset(void* dptr, int value, std::size_t count) {
+@@ -563,7 +563,7 @@ namespace strumpack {
+       STRUMPACK_FLOPS(4*blas::trsm_flops(m, n, alpha, side));
+       STRUMPACK_BYTES(2*8*blas::trsm_moves(m, n));
+       gpu_check(hipblasZtrsm(handle, side, uplo, trans, diag, m, n,
+-                             reinterpret_cast<const hipblasDoubleComplex*>(alpha), 
++                             reinterpret_cast<const hipblasDoubleComplex*>(alpha),
+                              reinterpret_cast<hipblasDoubleComplex*>(A), lda,
+                              reinterpret_cast<hipblasDoubleComplex*>(B), ldb));
+     }