Building and running on perlmutter

Building

1. Load necessary module:

   module load cray-fftw

2. Create script called install.sh with the following contents:

   #!/bin/bash -e
   
   # kokkos
   git clone -b 4.2.00 https://github.com/kokkos/kokkos.git
   cmake -S kokkos -B build-kokkos \
     -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
     -DCMAKE_BUILD_TYPE="Release" \
     -DCMAKE_CXX_COMPILER=$PWD/kokkos/bin/nvcc_wrapper \
     -DKokkos_ARCH_AMPERE80=ON \
     -DKokkos_ENABLE_SERIAL=ON \
     -DKokkos_ENABLE_OPENMP=off \
     -DKokkos_ENABLE_CUDA=on \
     -DKokkos_ENABLE_CUDA_LAMBDA=on \
     -DKokkos_ENABLE_CUDA_CONSTEXPR=on \
     -DKokkos_ENABLE_DEBUG=off
   cmake --build build-kokkos -j 24 --target install
   
   # ADIOS2
   git clone [email protected]:ornladios/ADIOS2.git
   cmake -S ADIOS2/ -B build-ADIOS2 \
     -DCMAKE_INSTALL_PREFIX=build-ADIOS2/install \
     -DADIOS2_USE_CUDA=on \
     -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_CXX_COMPILER=CC \
     -DCMAKE_C_COMPILER=cc
   cmake --build build-ADIOS2 --target install -j8
   
   # perfstubs
   git clone [email protected]:UO-OACISS/perfstubs.git
   cmake -S perfstubs -B build-perfstubs \
     -DCMAKE_CXX_COMPILER=CC \
     -DCMAKE_C_COMPILER=cc \
     -DCMAKE_INSTALL_PREFIX=build-perfstubs/install
   cmake --build build-perfstubs -j2 --target install
   
   # redev
   git clone [email protected]:SCOREC/redev.git
   cmake -S redev -B build-redev \
     -DCMAKE_CXX_COMPILER=CC \
     -DCMAKE_C_COMPILER=cc \
     -DMPIEXEC_EXECUTABLE=`which srun` \
     -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_INSTALL_PREFIX=build-redev/install \
     -Dperfstubs_DIR=$PWD/build-perfstubs \
     -DADIOS2_ROOT=build-ADIOS2/install
   cmake --build build-redev -j2 --target install
   
   # omega_h
   git clone [email protected]:SCOREC/omega_h.git
   cmake -S omega_h -B build-omega_h \
     -DCMAKE_INSTALL_PREFIX=build-omega_h/install \
     -DCMAKE_BUILD_TYPE=Release \
     -DBUILD_SHARED_LIBS=off \
     -DOmega_h_USE_Kokkos=ON \
     -DOmega_h_USE_CUDA=on \
     -DOmega_h_CUDA_ARCH=80 \
     -DOmega_h_USE_MPI=on  \
     -DMPIEXEC_EXECUTABLE=srun \
     -DBUILD_TESTING=off  \
     -DCMAKE_C_COMPILER=cc \
     -DCMAKE_CXX_COMPILER=CC \
     -DKokkos_PREFIX=build-kokkos/install/lib64/cmake
   cmake --build build-omega_h -j24 --target install
   
   # Catch2
   git clone https://github.com/catchorg/Catch2
   cmake -S Catch2 -B build-Catch2 \
     -DCMAKE_INSTALL_PREFIX=$PWD/build-Catch2/install \
     -DCMAKE_CXX_COMPILER=CC
   cmake --build build-Catch2 -j2 --target install
   
   # pcms
   
   git clone [email protected]:jacobmerson/pcms_testcases.git
   git clone https://github.com/SCOREC/pcms.git
   cmake -S pcms -B build-pcms \
     -DCMAKE_C_COMPILER=cc \
     -DCMAKE_CXX_COMPILER=CC \
     -DCMAKE_BUILD_TYPE=Release \
     -Dperfstubs_DIR=$PWD/build-perfstubs \
     -Dredev_DIR=$PWD/build-redev/install/lib64/cmake/redev \
     -DOmega_h_DIR=$PWD/build-omega_h/install/lib64/cmake/Omega_h/ \
     -DKokkos_DIR=$PWD/build-kokkos/install/lib64/cmake/Kokkos/ \
     -DCatch2_DIR=$PWD/build-Catch2/install/lib64/cmake/Catch2/ \
     -DPCMS_TEST_DATA_DIR=$PWD/pcms_testcases
   cmake --build build-pcms -j8
   

3. Make script executable:

   chmod +x install.sh

4. Run install script:

   ./install.sh

Running

1. Create allocation, make sure to add valid account:

   salloc --time 00:20:00 --constrain=gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=mXXXX 
   

2. Navigate to build dir:

   cd build-pcms

3. Run ctest