From 153763941ccea33539a97a274f4766c4cf20fc0d Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 22 Aug 2024 08:48:44 -0700
Subject: [PATCH] updates for perlmutter

---
 machines/perlmutter/config_batch.xml    |  9 ++--
 machines/perlmutter/config_machines.xml | 65 +++++++++++++++----------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/machines/perlmutter/config_batch.xml b/machines/perlmutter/config_batch.xml
index ed14205a..e5b9e2a2 100644
--- a/machines/perlmutter/config_batch.xml
+++ b/machines/perlmutter/config_batch.xml
@@ -7,11 +7,14 @@
       <argument> --account $PROJECT </argument>
     </submit_args>
     <directives>
-      <directive>-C gpu </directive>
+      <directive>--constraint=cpu </directive>
     </directives>
     <queues>
-      <queue walltimemax="06:00:00" nodemin="1" nodemax="128">regular</queue>
-      <queue walltimemax="02:00:00" nodemin="1" nodemax="4" default="true">debug</queue>
+      <!-- Note: walltime is not the max walltime, but the default - see NERSC docs for Q limits, https://docs.nersc.gov/jobs/policy/ -->
+      <queue walltimemax="00:30:00" nodemax="3072" default="true">regular</queue>
+      <queue walltimemax="00:30:00" nodemax="3072" strict="true">preempt</queue>
+      <queue walltimemax="00:30:00" nodemax="3072" strict="true">shared</queue>
+      <queue walltimemax="00:30:00" nodemax="8" strict="true">debug</queue>
     </queues>
   </batch_system>
 
diff --git a/machines/perlmutter/config_machines.xml b/machines/perlmutter/config_machines.xml
index fac14222..7bde9649 100644
--- a/machines/perlmutter/config_machines.xml
+++ b/machines/perlmutter/config_machines.xml
@@ -1,7 +1,7 @@
   <machine MACH="perlmutter">
     <DESC>NERSC EX AMD EPYC, os is CNL, 64 pes/node, batch system is Slurm</DESC>
     <OS>CNL</OS>
-    <COMPILERS>gnu,cray,nvidia,aocc</COMPILERS>
+    <COMPILERS>intel,gnu,cray,nvidia,aocc</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>mp9_g</PROJECT>
     <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
@@ -25,7 +25,7 @@
         <arg name="binding"> -c {{ srun_binding }}</arg>
       </arguments>
     </mpirun>
-    <module_system type="module">
+    <module_system type="module" allow_error="true">
       <init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
       <init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
       <init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
@@ -35,24 +35,36 @@
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <modules>
-        <command name="rm">PrgEnv-nvidia</command>
-        <command name="rm">PrgEnv-cray</command>
-        <command name="rm">PrgEnv-aocc</command>
-        <command name="rm">PrgEnv-gnu</command>
-        <command name="rm">nvidia</command>
-        <command name="rm">cce</command>
-        <command name="rm">gnu</command>
-        <command name="rm">aocc</command>
-        <command name="rm">cray-parallel-netcdf</command>
-        <command name="rm">cray-hdf5-parallel</command>
-        <command name="rm">cray-libsci</command>
-        <command name="rm">cray-mpich</command>
-        <command name="rm">cray-hdf5</command>
-        <command name="rm">cray-netcdf-hdf5parallel</command>
-        <command name="rm">cray-netcdf</command>
-        <command name="rm">craype</command>
+        <command name="unload">cray-hdf5-parallel</command>
+        <command name="unload">cray-netcdf-hdf5parallel</command>
+        <command name="unload">cray-parallel-netcdf</command>
+        <command name="unload">cray-netcdf</command>
+        <command name="unload">cray-hdf5</command>
+        <command name="unload">PrgEnv-gnu</command>
+        <command name="unload">PrgEnv-intel</command>
+        <command name="unload">PrgEnv-nvidia</command>
+        <command name="unload">PrgEnv-cray</command>
+        <command name="unload">PrgEnv-aocc</command>
+        <command name="unload">gcc-native</command>
+        <command name="unload">intel</command>
+        <command name="unload">intel-oneapi</command>
+        <command name="unload">nvidia</command>
+        <command name="unload">aocc</command>
+        <command name="unload">cudatoolkit</command>
+        <command name="unload">climate-utils</command>
+        <command name="unload">cray-libsci</command>
+        <command name="unload">matlab</command>
+        <command name="unload">craype-accel-nvidia80</command>
+        <command name="unload">craype-accel-host</command>
+        <command name="unload">perftools-base</command>
+        <command name="unload">perftools</command>
+        <command name="unload">darshan</command>
       </modules>
 
+      <modules compiler="intel">
+	<command name="load">PrgEnv-intel</command>
+      </modules>
+      
       <modules compiler="cray">
         <command name="load">PrgEnv-cray</command>
         <command name="switch">cce cce/12.0.3</command>
@@ -62,10 +74,8 @@
 	
       </modules>
       <modules>
-        <command name="swap">craype craype/2.7.30</command>
-      </modules>
-      <modules>
-        <command name="switch">cray-libsci/23.12.5</command>
+	<command name="load">craype-x86-milan</command>
+        <command name="switch">cray-libsci/23.09.1.1</command>
       </modules>
       <modules>
         <command name="load">cray-mpich/8.1.28</command>
@@ -83,12 +93,17 @@
         <command name="load">cray-parallel-netcdf</command>
       </modules>
       <modules>
-        <command name="load">cmake/3.22.0</command>
+        <command name="load">cmake/3.24.3</command>
       </modules>
     </module_system>
     <environment_variables>
       <env name="OMP_STACKSIZE">256M</env>
-<!--      <env name="OMP_PROC_BIND">spread</env>
-      <env name="OMP_PLACES">threads</env> -->
+      <env name="ESMFMKFILE">/global/cfs/cdirs/ccsm1/esmf//lib/libO/Unicos.intel.64.mpi.default/esmf.mk</env>
+      <env name="PIO_VERSION_MAJOR">2</env>
+      <env name="PIO_LIBDIR">/global/cfs/cdirs/ccsm1/parallelio/2.6.3/2023.2.0/8.1.28/lib</env>
+      <env name="PIO_INCDIR">/global/cfs/cdirs/ccsm1/parallelio/2.6.3/2023.2.0/8.1.28/include</env>
+      <env name="PIO_TYPENAME_VALID_VALUES">pnetcdf,netcdf,netcdf4p</env>
+      <env name="ESMF_RUNTIME_PROFILE">ON</env>
+      <env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
     </environment_variables>
   </machine>