Small updates

xiaoyeli · May 12, 2021 · cba813d · cba813d
1 parent 331c3a3
commit cba813d
Show file tree

Hide file tree

Showing 18 changed files with 34 additions and 40 deletions.
diff --git a/EXAMPLE/Makefile b/EXAMPLE/Makefile
@@ -30,7 +30,7 @@
 #######################################################################
 include ../make.inc
 
-DEXM	= pddrive.o dcreate_matrix.o sp_ienv.o 
+DEXM	= pddrive.o dcreate_matrix.o 
 	#pdgssvx.o pdgstrf2.o
 DEXM1	= pddrive1.o dcreate_matrix.o
 DEXM2	= pddrive2.o dcreate_matrix.o dcreate_matrix_perturbed.o

diff --git a/FORTRAN/f_5x5.F90 b/FORTRAN/f_5x5.F90
@@ -33,7 +33,7 @@ program f_5x5
 !   6. Release the process grid and terminate the MPI environment
 !   7. Release all structures
 !
-      #include "superlu_dist_config.fh"
+#include "superlu_dist_config.fh"
       use superlu_mod
       include 'mpif.h'
       integer maxn, maxnz, maxnrhs

diff --git a/FORTRAN/f_pddrive.F90 b/FORTRAN/f_pddrive.F90
@@ -29,7 +29,7 @@ program f_pddrive
 !   7. Release all structures
 !
 !
-      #include "superlu_dist_config.fh"
+#include "superlu_dist_config.fh"
       use superlu_mod
       include 'mpif.h'
       integer maxn, maxnz, maxnrhs

diff --git a/FORTRAN/f_pddrive3d.F90 b/FORTRAN/f_pddrive3d.F90
@@ -41,7 +41,7 @@ program f_pddrive3d
 ! The program may be run by typing
 !    mpiexec -np 8 f_pddrive3d 
 !
-      #include "superlu_dist_config.fh"
+#include "superlu_dist_config.fh"
       use superlu_mod
 !      implicit none
       include 'mpif.h'

diff --git a/FORTRAN/f_pzdrive.F90 b/FORTRAN/f_pzdrive.F90
@@ -28,7 +28,7 @@ program f_pzdrive
 !   7. Release all structures
 !
 !
-      #include "superlu_dist_config.fh"
+#include "superlu_dist_config.fh"
       use superlu_mod
       include 'mpif.h'
       integer maxn, maxnz, maxnrhs

diff --git a/FORTRAN/f_pzdrive3d.F90 b/FORTRAN/f_pzdrive3d.F90
@@ -40,7 +40,7 @@ program f_pzdrive3d
 ! The program may be run by typing
 !    mpiexec -np 8 f_pzdrive3d 
 !
-      #include "superlu_dist_config.fh"
+#include "superlu_dist_config.fh"
       use superlu_mod
 !      implicit none
       include 'mpif.h'

diff --git a/FORTRAN/sp_ienv.c b/FORTRAN/sp_ienv.c
@@ -53,6 +53,8 @@ at the top-level directory.
 	         of L and U, compared with A;
 	    = 7: the minimum value of the product M*N*K for a GEMM call
 	         to be off-loaded to accelerator (e.g., GPU, Xeon Phi).
+            = 8: the maximum buffer size on GPU that can hold the three
+	         matrices in the GEMM call for the Schur complement update.
 	    
    (SP_IENV_DIST) (output) int
             >= 0: the value of the parameter specified by ISPEC   
@@ -62,11 +64,9 @@ at the top-level directory.
 </pre>
 */
 
-
 #include <stdlib.h>
 #include <stdio.h>
 
-
 int
 sp_ienv_dist(int ispec)
 {
@@ -94,28 +94,33 @@ sp_ienv_dist(int ispec)
             return 1;
 
 	case 3: 
-            ttemp = getenv("NSUP");
+	    ttemp = getenv("NSUP"); // take min of MAX_SUPER_SIZE in superlu_defs.h
             if(ttemp)
             {
-                return(atoi(ttemp));
+		int k = SUPERLU_MIN( atoi(ttemp), MAX_SUPER_SIZE );
+                return (k);
             }
-            else
-            return 128;
+            else return 128;
 
 #endif
-        case 6: return (5);
+        case 6: 
+            ttemp = getenv("FILL");
+            if ( ttemp ) return(atoi(ttemp));
+            else return (5);
         case 7:
 	    ttemp = getenv ("N_GEMM");
 	    if (ttemp) return atoi (ttemp);
 	    else return 10000;
-
+        case 8:
+  	    ttemp = getenv ("MAX_BUFFER_SIZE");
+	    if (ttemp) return atoi (ttemp);
+	    else return 64000000; // 8000^2
     }
 
     /* Invalid value for ISPEC */
     i = 1;
     xerr_dist("sp_ienv", &i);
     return 0;
 
-
 } /* sp_ienv_dist */
 
diff --git a/FORTRAN/superlu_dist_config.fh b/FORTRAN/superlu_dist_config.fh
@@ -1,11 +1,9 @@
 
-#define HAVE_CUDA TRUE
 
 #define HAVE_PARMETIS TRUE
 
 
 
-#define XSDK_INDEX_SIZE 64
 
 #if (XSDK_INDEX_SIZE == 64)
 #define _LONGINT 1

diff --git a/SRC/dscatter3d.c b/SRC/dscatter3d.c
@@ -19,7 +19,6 @@ at the top-level directory.
 #else
 //#include "cblas.h"
 #endif
-#include "omp.h"
 
 #define ISORT
 #define SCATTER_U_CPU  scatter_u

diff --git a/SRC/dsuperlu_gpu.cu b/SRC/dsuperlu_gpu.cu
@@ -14,7 +14,6 @@
 //#define GPU_DEBUG
 
 #include "mpi.h"
-#include "omp.h"
 // #include "sec_structs.h"
 #include <ctime>
 #include <cublas_v2.h>
@@ -98,9 +97,7 @@ void device_scatter_l (int_t thread_id,
 }
 #endif ///////////// not used
 
-#define THREAD_BLOCK_SIZE  256  /* Sherry: was 192. should be <= MAX_SUPER_SIZE */
-// The following is moved to superlu_defs.h
-//#define MAX_SUPER_SIZE   256    /* Sherry: was 192 on Titan */
+#define THREAD_BLOCK_SIZE  512  /* Sherry: was 192. should be <= MAX_SUPER_SIZE */
 
 __device__ inline
 void ddevice_scatter_l_2D (int thread_id,

diff --git a/SRC/scatter.c b/SRC/scatter.c
@@ -7,7 +7,7 @@
 #else
 #include "cblas.h"
 #endif
-#include "omp.h"
+
 
 #define ISORT
 

diff --git a/SRC/sec_structs.c b/SRC/sec_structs.c
@@ -4,7 +4,6 @@
 #include "sec_structs.h"
 #include <stdlib.h> /*for Qsort */
 #include <mpi.h>
-#include <omp.h>
 #include <math.h>   /*for sqrt*/
 #include <string.h>
 #include "compiler.h"

diff --git a/SRC/sp_ienv.c b/SRC/sp_ienv.c
@@ -94,13 +94,13 @@ sp_ienv_dist(int ispec)
             return 20;
 
 	case 3: 
-	  ttemp = getenv("NSUP"); // take min of MAX_SUPER_SIZE in superlu_defs.h
+	    ttemp = getenv("NSUP"); // take min of MAX_SUPER_SIZE in superlu_defs.h
             if(ttemp)
             {
-                return(atoi(ttemp));
+		int k = SUPERLU_MIN( atoi(ttemp), MAX_SUPER_SIZE );
+                return (k);
             }
-            else
-            return 128;
+            else return 128;
 
 #endif
         case 6: 

diff --git a/SRC/superlu_FortranCInterface.h b/SRC/superlu_FortranCInterface.h
@@ -2,15 +2,15 @@
 #define FC_HEADER_INCLUDED
 
 /* Mangling for Fortran global symbols without underscores. */
-#define FC_GLOBAL(name,NAME) name
+#define FC_GLOBAL(name,NAME) name##_
 
 /* Mangling for Fortran global symbols with underscores. */
-#define FC_GLOBAL_(name,NAME) name
+#define FC_GLOBAL_(name,NAME) name##_
 
 /* Mangling for Fortran module symbols without underscores. */
-#define FC_MODULE(mod_name,name, mod_NAME,NAME) __##mod_name##_NMOD_##name
+#define FC_MODULE(mod_name,name, mod_NAME,NAME) __##mod_name##_MOD_##name
 
 /* Mangling for Fortran module symbols with underscores. */
-#define FC_MODULE_(mod_name,name, mod_NAME,NAME) __##mod_name##_NMOD_##name
+#define FC_MODULE_(mod_name,name, mod_NAME,NAME) __##mod_name##_MOD_##name
 
 #endif
diff --git a/SRC/superlu_defs.h b/SRC/superlu_defs.h
@@ -136,7 +136,7 @@ typedef MPI_C_DOUBLE_COMPLEX  SuperLU_MPI_DOUBLE_COMPLEX;
 #endif
 
 
-#define MAX_SUPER_SIZE 256   /* Sherry: moved from superlu_gpu.cu */
+#define MAX_SUPER_SIZE 512   /* Sherry: moved from superlu_gpu.cu */
 
 
 #define ISORT     /* NOTE: qsort() has bug on Mac */

diff --git a/SRC/superlu_dist_config.h b/SRC/superlu_dist_config.h
@@ -1,19 +1,19 @@
 /* superlu_dist_config.h.in */
 
 /* Enable CUDA */
-#define HAVE_CUDA TRUE
+/* #undef HAVE_CUDA */
 
 /* Enable parmetis */
 #define HAVE_PARMETIS TRUE
 
 /* Enable LAPACK */
-#define SLU_HAVE_LAPACK TRUE
+/* #undef SLU_HAVE_LAPACK */
 
 /* Enable CombBLAS */
 /* #undef HAVE_COMBBLAS */
 
 /* enable 64bit index mode */
-#define XSDK_INDEX_SIZE 64
+/* #undef XSDK_INDEX_SIZE */
 
 #if (XSDK_INDEX_SIZE == 64)
 #define _LONGINT 1

diff --git a/SRC/zscatter3d.c b/SRC/zscatter3d.c
@@ -18,7 +18,6 @@ at the top-level directory.
 #else
 //#include "cblas.h"
 #endif
-#include "omp.h"
 
 #define ISORT
 #define SCATTER_U_CPU  scatter_u

diff --git a/SRC/zsuperlu_gpu.cu b/SRC/zsuperlu_gpu.cu
@@ -13,7 +13,6 @@
 //#define GPU_DEBUG
 
 #include "mpi.h"
-#include "omp.h"
 // #include "sec_structs.h"
 #include <ctime>
 #include <cublas_v2.h>
@@ -84,8 +83,6 @@ void device_scatter_l (int_t thread_id,
 #endif ///////////// not used
 
 #define THREAD_BLOCK_SIZE  256  /* Sherry: was 192. should be <= MAX_SUPER_SIZE */
-// The following is moved to superlu_defs.h
-//#define MAX_SUPER_SIZE   256    /* Sherry: was 192 on Titan */
 
 __device__ inline
 void zdevice_scatter_l_2D (int thread_id,