Merge pull request #97 from sjsprecious/add_gpu_gust

Add flexible controls of GPU configuration and remove old GPU configuration

machines/Depends.nvhpc

@@ -67,11 +67,7 @@ mo_rte_kind.o\
 mo_source_functions.o\
 mo_optical_props_kernels.o\
 mo_rte_solver_kernels.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-rrtmgp_driver.o
-
-RRTMGP_NOOPT=\
+rrtmgp_driver.o\
 mo_gas_optics_kernels.o
 
 ifeq ($(DEBUG),FALSE)
@@ -86,9 +82,7 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
   $(RRTMGP_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $<
-  $(RRTMGP_NOOPT): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
 endif

machines/cmake_macros/nvhpc.cmake

@@ -49,3 +49,21 @@ set(MPIFC "mpif90")
 set(SCC "nvc")
 set(SCXX "nvc++")
 set(SFC "nvfortran")
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp)
+   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp)
+   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel")
+endif()

machines/cmake_macros/nvhpc_casper.cmake

@@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -ldl")
 endif()
 string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")

machines/cmake_macros/nvhpc_derecho.cmake

@@ -4,3 +4,4 @@ endif()
 if (NOT DEBUG)
   string(APPEND FFLAGS " -target=zen3")
 endif()
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")

machines/config_batch.xml

@@ -156,7 +156,6 @@
       <directive default="n"> -r {{ rerunnable }} </directive>
       <!-- <directive> -j oe {{ job_id }} </directive> -->
       <directive> -j oe </directive>
-      <directive> -V </directive>
     </directives>
   </batch_system>
 
@@ -226,34 +225,14 @@
   <!-- casper pbs -->
   <batch_system MACH="casper" type="pbs">
     <batch_submit>qsub</batch_submit>
-
-    <directives queue="casper" compiler="pgi-gpu">
-      <!-- Turn on MPS server manually -->
-      <!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
-      <directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-      <directive> -l gpu_type=v100 </directive>
-    </directives>
-
-    <directives queue="casper" compiler="nvhpc-gpu">
-      <!-- Turn on MPS server manually -->
-      <!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
-      <directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-      <directive> -l gpu_type=v100 </directive>
-    </directives>
-
-    <directives queue="casper" compiler="nvhpc">
-      <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-    </directives>
-
-    <directives queue="casper" compiler="pgi">
+    <submit_args>
+      <argument> -l gpu_type=$GPU_TYPE </argument>
+    </submit_args>
+    <directives queue="casper" compiler="nvhpc" gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
-
-    <directives queue="casper" compiler="intel">
+    <directives queue="casper" gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
     </directives>
@@ -378,9 +357,13 @@
 
   <batch_system MACH="derecho" type="pbs" >
     <batch_submit>qsub</batch_submit>
-    <directives>
+    <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB</directive>
+    </directives>
+    <directives gpu_enabled="true">
+      <directive default="/bin/bash" > -S {{ shell }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="2488" >main</queue>
@@ -501,10 +484,19 @@
 
   <batch_system MACH="gust" type="pbs" >
     <batch_submit>qsub</batch_submit>
-    <directives>
+    <submit_args>
+      <argument> -l gpu_type=$GPU_TYPE </argument>
+    </submit_args>
+    <directives queue="main" gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
     </directives>
+    <directives queue="main" gpu_enabled="true">
+      <directive default="/bin/bash" > -S {{ shell }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} </directive>
+      <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
+      <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
+    </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="8" >main</queue>
       <queue walltimemax="1:00:00" nodemin="9" nodemax="16" >bigcpu</queue>