Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add tacc vista system and update stampede #188

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

add tacc vista system and update stampede #188

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a2b7ac2
add tacc vista system and update stampede
jedwards4b Sep 18, 2024
a9f4786
update cprnc path
jedwards4b Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions machines/cmake_macros/intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ endif()
set(MPICC "mpicc")
set(MPICXX "mpicxx")
set(MPIFC "mpif90")
set(SCC "icc")
set(SCXX "icpc")
set(SCC "icx")
set(SCXX "icpx")
set(SFC "ifort")
if (MPILIB STREQUAL mpich)
string(APPEND SLIBS " -mkl=cluster")
Expand Down
2 changes: 2 additions & 0 deletions machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,13 @@ This allows using a different mpirun command to launch unit tests
<value MACH="sandia-srn-sems">(s999964|climate|penn)</value>
<value MACH="sandiatoss3">(skybridge|chama)-login</value>
<value MACH="stampede2-skx">.*stampede2</value>
<value MACH="stampede3-spr">.*.stampede3.tacc.utexas.edu</value>
<value MACH="swan">swan.*</value>
<value MACH="theia">tfe</value>
<value MACH="theta">theta.*</value>
<value MACH="thunder">.*.thunder.ucar.edu</value>
<value MACH="ubuntu-latest">$ENV{CIME_TEST_PLATFORM}:ubuntu-latest</value>
<value MACH="vista">.*\.vista.tacc.utexas.edu</value>
<value MACH="zeus">(login[1,2]-ib|n[0-9][0-9][0-9]-ib)</value>
</NODENAME_REGEX>

Expand Down
11 changes: 11 additions & 0 deletions machines/stampede3-spr/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<batch_system MACH="stampede3-spr" type="slurm" >
<batch_submit>ssh login1 cd $CASEROOT ; sbatch</batch_submit>
<submit_args>
<argument> --time $JOB_WALLCLOCK_TIME </argument>
<argument> -p $JOB_QUEUE </argument>
<argument> --account $PROJECT </argument>
</submit_args>
<queues>
<queue walltimemax="24:00:00" nodemin="1" nodemax="16" default="true">spr</queue>
</queues>
</batch_system>
70 changes: 70 additions & 0 deletions machines/stampede3-spr/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
<machine MACH="stampede3-spr">
<DESC>Intel Xeon CPU MAX 9480 ("Sapphire Rapids HBM") 112 cores on two sockets (2x 56 cores), batch system is SLURM</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>impi,mvapich</MPILIBS>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/work/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/work/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$ENV{WORK}/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/work/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>/work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.sp3</CCSM_CPRNC>
<GMAKE_J>4</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>cseg</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>112</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="impi">
<executable>ibrun</executable>
<arguments>
<arg name="ntasks"> -n {{ total_tasks }} </arg>
</arguments>
</mpirun>
<mpirun mpilib="mvapich2">
<executable>ibrun</executable>
<arguments>
<arg name="ntasks"> -n {{ total_tasks }} </arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
<init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
<cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"></command>
<command name="load">TACC</command>
<command name="load">intel/24.0</command>
<command name="load">cmake/3.28.1</command>
</modules>
<modules mpilib="mvapich2">
<command name="load">mvapich/3.0</command>
</modules>
<modules mpilib="impi">
<command name="load">impi</command>
</modules>
<modules mpilib="!mpi-serial">
<command name="load">pnetcdf/1.12.3</command>
<command name="load">parallel-netcdf/4.9.2</command>
<command name="load">phdf5/1.14.3</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">netcdf/4.9.2</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="ESMFMKFILE">/work2/02503/edwardsj/stampede3/intel24.0/esmf/v8.6.1/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk</env>
<env name="PIO_VERSION_MAJOR">2</env>
<env name="PIO_LIBDIR">/work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/lib</env>
<env name="PIO_INCDIR">/work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/include</env>
<env name="PIO_TYPENAME_VALID_VALUES">netcdf,pnetcdf,netcdf4p,netcdf4c</env>
</environment_variables>
<environment_variables mpilib="impi">
<env name="I_MPI_F90">ifort</env>
</environment_variables>
</machine>
11 changes: 11 additions & 0 deletions machines/stampede3-spr/intel_stampede3-spr.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
if (MPILIB STREQUAL mpi-serial)
string(APPEND FFLAGS " -mcmodel medium")
endif()
string(APPEND CFLAGS " -xCORE-AVX512")
string(APPEND FFLAGS " -xCORE-AVX512")

string(APPEND LDFLAGS " -L$ENV{TACC_HDF5_LIB} -lhdf5 $(MKL)")
execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -L$ENV{TACC_HDF5_LIB} -lhdf5")
set(TRILINOS_PATH "$ENV{TRILINOS_PATH}")
set(HAS_F2008_CONTIGUOUS "FALSE")
5 changes: 5 additions & 0 deletions machines/stampede3-spr/stampede3-spr.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
string(APPEND CPPDEFS " -DHAVE_NANOTIME")
set(NETCDF_PATH "$ENV{TACC_NETCDF_DIR}")
set(PIO_FILESYSTEM_HINTS "lustre")
set(PNETCDF_PATH "$ENV{TACC_PNETCDF_DIR}")
string(APPEND SLIBS " -Wl,-rpath,$ENV{PIO_LIBDIR}")
12 changes: 12 additions & 0 deletions machines/vista/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<batch_system MACH="vista" type="slurm" >
<batch_submit>ssh vista.tacc.utexas.edu cd $CASEROOT ; sbatch</batch_submit>
<submit_args>
<argument> --time $JOB_WALLCLOCK_TIME </argument>
<argument> -p $JOB_QUEUE </argument>
<argument> --account $PROJECT </argument>
</submit_args>
<queues>
<queue walltimemax="48:00:00" nodemin="1" nodemax="256" default="true">gg</queue>
<queue walltimemax="02:00:00" nodemin="1" nodemax="4" >gg-4k</queue>
</queues>
</batch_system>
60 changes: 60 additions & 0 deletions machines/vista/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
<machine MACH="vista">
<DESC>Grace Hopper ARM TACC , batch system is SLURM</DESC>
<OS>LINUX</OS>
<COMPILERS>nvhpc,gnu</COMPILERS>
<MPILIBS>openmpi</MPILIBS>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/work2/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/work2/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/work2/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>/work2/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.vista</CCSM_CPRNC>
<GMAKE_J>4</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY></SUPPORTED_BY>
<MAX_TASKS_PER_NODE>144</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>144</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="openmpi">
<executable>mpirun</executable>
<arguments>
<arg name="num_tasks"> -np {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
<init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
<cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"/>
<command name="load">nvidia/24.7</command>
<command name="load">ucx/1.17.0</command>
<command name="load">openmpi/5.0.5</command>
<command name="load">pnetcdf/1.13.0</command>
<command name="load">hdf5/1.14.4</command>
<command name="use">/work/02503/edwardsj/vista/modulefiles/nvhpc/24.7.0/openmpi/5.0.5</command>
<command name="load">parallelio/2.6.3</command>
<command name="load">esmf/8.7.0b14</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="ESMF_RUNTIME_PROFILE">ON</env>
<env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
</environment_variables>
<environment_variables compiler="nvhpc">
<env name="NETCDF_PATH">/scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_nvhpc_24.7</env>
</environment_variables>
<environment_variables compiler="gnu">
<env name="PIO_LIBDIR">/work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/lib</env>
<env name="PIO_INCDIR">/work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/include</env>
<env name="NETCDF_PATH">/scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_gcc_14.2</env>
<env name="PNETCDF_PATH">/scratch/00422/cazes/pnetcdf_1.13.0_gcc_14.2/</env>
<env name="ESMFMKFILE">/work/02503/edwardsj/vista/esmf/v8.6.2b00_gnu_openmpi/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk</env>
<env name="LD_LIBRARY_PATH">/home1/apps/nvidia/Linux_aarch64/24.7/compilers/lib:/scratch/projects/compilers/gcc11/openmpi/5.0.3_cpu/lib:/scratch/projects/compilers/ucx/1.17/lib/ucx:/scratch/projects/compilers/ucx/1.17/lib:/opt/apps/gcc/14.2.0/lib64:/opt/apps/gcc/14.2.0/lib</env>
</environment_variables>
</machine>
5 changes: 5 additions & 0 deletions machines/vista/nvhpc_vista.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
set(SUPPORTS_CXX "TRUE")
string(APPEND SLIBS " -L$ENV{NETCDFHOME}/lib -lnetcdff -lnetcdf ")
if (MPILIB STREQUAL openmpi)
string(APPEND SLIBS " -L$ENV{PNETCDFHOME}/lib -lpnetcdf ")
endif()
3 changes: 3 additions & 0 deletions machines/vista/vista.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
string(APPEND LDFLAGS " -Wl,-rpath,/scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_nvhpc_24.7/lib")
string(APPEND LDFLAGS " -Wl,-rpath,/scratch/00422/cazes/pnetcdf_1.13.0_nvhpc/lib")
set(LAPACK_LIBDIR "/home1/apps/nvidia/Linux_aarch64/24.7/compilers/lib")