GPU update: add superlu_gpu_util.cu;

    Test if (superlu_acc_offload) then free_LUstruct_gpu;
    Allow optionally use round-robin binding of MPI ranks to GPU.

Author: xiaoyeli

EXAMPLE/pddrive.c

@@ -69,7 +69,6 @@ int main(int argc, char *argv[])
     nprow = 1;  /* Default process rows.      */
     npcol = 1;  /* Default process columns.   */
     nrhs = 1;   /* Number of right-hand side. */
-    printf("MAIN ...\n"); fflush(stdout);
 
     /* ------------------------------------------------------------
        INITIALIZE MPI ENVIRONMENT. 

SRC/CMakeLists.txt

@@ -56,7 +56,7 @@ set(sources
   sec_structs.c
 )
 if (HAVE_CUDA)
-  list(APPEND sources cublas_utils.c)
+  list(APPEND sources cublas_utils.c superlu_gpu_utils.cu)
 endif()
 
 if (MSVC)

SRC/Makefile

@@ -85,7 +85,7 @@ ZPLUSRC += pzgssvx3d.o pzgstrf3d.o ztreeFactorization.o zscatter3d.o \
 	znrformat_loc3d.o ztreeFactorizationGPU.o ##$(FACT3D)
 
 ifeq ($(HAVE_CUDA),TRUE)
-ALLAUX += cublas_utils.o
+ALLAUX += cublas_utils.o superlu_gpu_utils.o
 DPLUSRC += dsuperlu_gpu.o
 ZPLUSRC += zsuperlu_gpu.o
 endif

SRC/pdgstrf3d.c

@@ -344,7 +344,9 @@ int_t pdgstrf3d(superlu_dist_options_t *options, int m, int n, double anorm,
 
 #ifdef GPU_ACC
     /* This frees the GPU storage allocateed in initSluGPU3D_t() */
-    dfree_LUstruct_gpu (sluGPU->A_gpu);
+    if (superlu_acc_offload) {
+         dfree_LUstruct_gpu (sluGPU->A_gpu);
+    }
 #endif
 
     MPI_Barrier( grid3d->comm);

SRC/psgstrf3d.c

@@ -344,7 +344,9 @@ int_t psgstrf3d(superlu_dist_options_t *options, int m, int n, float anorm,
 
 #ifdef GPU_ACC
     /* This frees the GPU storage allocateed in initSluGPU3D_t() */
-    sfree_LUstruct_gpu (sluGPU->A_gpu);
+    if (superlu_acc_offload) {
+         sfree_LUstruct_gpu (sluGPU->A_gpu);
+    }
 #endif
 
     MPI_Barrier( grid3d->comm);

SRC/pzgstrf3d.c

@@ -343,7 +343,9 @@ int_t pzgstrf3d(superlu_dist_options_t *options, int m, int n, double anorm,
 
 #ifdef GPU_ACC
     /* This frees the GPU storage allocateed in initSluGPU3D_t() */
-    zfree_LUstruct_gpu (sluGPU->A_gpu);
+    if (superlu_acc_offload) {
+         zfree_LUstruct_gpu (sluGPU->A_gpu);
+    }
 #endif
 
     MPI_Barrier( grid3d->comm);

SRC/sp_ienv.c

@@ -116,7 +116,7 @@ sp_ienv_dist(int ispec)
         case 8:
   	    ttemp = getenv ("MAX_BUFFER_SIZE");
 	    if (ttemp) return atoi (ttemp);
-	    else return 1000000000; // 256000000 = 16000^2
+	    else return 256000000; // 256000000 = 16000^2
     }
 
     /* Invalid value for ISPEC */

SRC/superlu_grid.c

@@ -64,11 +64,15 @@ void superlu_gridinit(MPI_Comm Bcomm, /* The base communicator upon which
 
 #ifdef GPU_ACC
     /* Binding each MPI to a CUDA device */
-    int devs, rank;
-    MPI_Comm_rank(Bcomm, &rank); // MPI_COMM_WORLD??
-    cudaGetDeviceCount(&devs);  // Returns the number of compute-capable devices
-    cudaSetDevice(rank % devs); // Set device to be used for GPU executions
-    ////
+    char *ttemp;
+    ttemp = getenv ("SUPERLU_BIND_MPI_GPU");
+
+    if (ttemp) {
+	int devs, rank;
+	MPI_Comm_rank(Bcomm, &rank); // MPI_COMM_WORLD??
+	cudaGetDeviceCount(&devs);  // Returns the number of compute-capable devices
+	cudaSetDevice(rank % devs); // Set device to be used for GPU executions
+    }
 #endif
 }
 

SRC/superlu_grid3d.c

@@ -49,13 +49,16 @@ void superlu_gridinit3d(MPI_Comm Bcomm, /* The base communicator upon which
 
 #ifdef GPU_ACC
     /* Binding each MPI to a CUDA device */
-    int devs, rank;
-    MPI_Comm_rank(Bcomm, &rank); // MPI_COMM_WORLD??
-    cudaGetDeviceCount(&devs);  // Returns the number of compute-capable devices
-    cudaSetDevice(rank % devs); // Set device to be used for GPU executions
-    ////
+    char *ttemp;
+    ttemp = getenv ("SUPERLU_BIND_MPI_GPU");
+
+    if (ttemp) {
+	int devs, rank;
+	MPI_Comm_rank(Bcomm, &rank); // MPI_COMM_WORLD??
+	cudaGetDeviceCount(&devs);  // Returns the number of compute-capable devices
+	cudaSetDevice(rank % devs); // Set device to be used for GPU executions
+    }
 #endif
-    
 }