diff --git a/machines/cmake_macros/intel.cmake b/machines/cmake_macros/intel.cmake index 3c25605..d084a85 100644 --- a/machines/cmake_macros/intel.cmake +++ b/machines/cmake_macros/intel.cmake @@ -33,8 +33,8 @@ endif() set(MPICC "mpicc") set(MPICXX "mpicxx") set(MPIFC "mpif90") -set(SCC "icc") -set(SCXX "icpc") +set(SCC "icx") +set(SCXX "icpx") set(SFC "ifort") if (MPILIB STREQUAL mpich) string(APPEND SLIBS " -mkl=cluster") diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 2c0a847..60200d2 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -68,11 +68,13 @@ This allows using a different mpirun command to launch unit tests (s999964|climate|penn) (skybridge|chama)-login .*stampede2 + .*.stampede3.tacc.utexas.edu swan.* tfe theta.* .*.thunder.ucar.edu $ENV{CIME_TEST_PLATFORM}:ubuntu-latest + .*\.vista.tacc.utexas.edu (login[1,2]-ib|n[0-9][0-9][0-9]-ib) diff --git a/machines/stampede3-spr/config_batch.xml b/machines/stampede3-spr/config_batch.xml new file mode 100644 index 0000000..d255ef4 --- /dev/null +++ b/machines/stampede3-spr/config_batch.xml @@ -0,0 +1,11 @@ + + ssh login1 cd $CASEROOT ; sbatch + + --time $JOB_WALLCLOCK_TIME + -p $JOB_QUEUE + --account $PROJECT + + + spr + + diff --git a/machines/stampede3-spr/config_machines.xml b/machines/stampede3-spr/config_machines.xml new file mode 100644 index 0000000..eb5f917 --- /dev/null +++ b/machines/stampede3-spr/config_machines.xml @@ -0,0 +1,70 @@ + + Intel Xeon CPU MAX 9480 ("Sapphire Rapids HBM") 112 cores on two sockets (2x 56 cores), batch system is SLURM + LINUX + intel + impi,mvapich + $ENV{SCRATCH} + /work/02503/edwardsj/CESM/inputdata + /work/02503/edwardsj/CESM/inputdata/lmwg + $ENV{WORK}/archive/$CASE + /work/02503/edwardsj/CESM/cesm_baselines + /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.sp3 + 4 + slurm + cseg + 112 + 112 + + ibrun + + -n {{ total_tasks }} + + + + ibrun + + -n {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + TACC + intel/24.0 + cmake/3.28.1 + + + mvapich/3.0 + + + impi + + + pnetcdf/1.12.3 + parallel-netcdf/4.9.2 + phdf5/1.14.3 + + + netcdf/4.9.2 + + + + 256M + /work2/02503/edwardsj/stampede3/intel24.0/esmf/v8.6.1/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk + 2 + /work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/lib + /work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/include + netcdf,pnetcdf,netcdf4p,netcdf4c + + + ifort + + diff --git a/machines/stampede3-spr/intel_stampede3-spr.cmake b/machines/stampede3-spr/intel_stampede3-spr.cmake new file mode 100644 index 0000000..7c48ea2 --- /dev/null +++ b/machines/stampede3-spr/intel_stampede3-spr.cmake @@ -0,0 +1,11 @@ +if (MPILIB STREQUAL mpi-serial) + string(APPEND FFLAGS " -mcmodel medium") +endif() +string(APPEND CFLAGS " -xCORE-AVX512") +string(APPEND FFLAGS " -xCORE-AVX512") + +string(APPEND LDFLAGS " -L$ENV{TACC_HDF5_LIB} -lhdf5 $(MKL)") +execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) +string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -L$ENV{TACC_HDF5_LIB} -lhdf5") +set(TRILINOS_PATH "$ENV{TRILINOS_PATH}") +set(HAS_F2008_CONTIGUOUS "FALSE") diff --git a/machines/stampede3-spr/stampede3-spr.cmake b/machines/stampede3-spr/stampede3-spr.cmake new file mode 100644 index 0000000..7b4ba5d --- /dev/null +++ b/machines/stampede3-spr/stampede3-spr.cmake @@ -0,0 +1,5 @@ +string(APPEND CPPDEFS " -DHAVE_NANOTIME") +set(NETCDF_PATH "$ENV{TACC_NETCDF_DIR}") +set(PIO_FILESYSTEM_HINTS "lustre") +set(PNETCDF_PATH "$ENV{TACC_PNETCDF_DIR}") +string(APPEND SLIBS " -Wl,-rpath,$ENV{PIO_LIBDIR}") diff --git a/machines/vista/config_batch.xml b/machines/vista/config_batch.xml new file mode 100644 index 0000000..fa61db1 --- /dev/null +++ b/machines/vista/config_batch.xml @@ -0,0 +1,12 @@ + + ssh vista.tacc.utexas.edu cd $CASEROOT ; sbatch + + --time $JOB_WALLCLOCK_TIME + -p $JOB_QUEUE + --account $PROJECT + + + gg + gg-4k + + diff --git a/machines/vista/config_machines.xml b/machines/vista/config_machines.xml new file mode 100644 index 0000000..292b38a --- /dev/null +++ b/machines/vista/config_machines.xml @@ -0,0 +1,60 @@ + + Grace Hopper ARM TACC , batch system is SLURM + LINUX + nvhpc,gnu + openmpi + $ENV{SCRATCH} + /work2/02503/edwardsj/CESM/inputdata + /work2/02503/edwardsj/CESM/inputdata/lmwg + $CIME_OUTPUT_ROOT/archive/$CASE + /work2/02503/edwardsj/CESM/cesm_baselines + /work2/02503/edwardsj/vista/cprnc/cprnc + 4 + slurm + + 144 + 144 + + mpirun + + -np {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + nvidia/24.7 + ucx/1.17.0 + openmpi/5.0.5 + pnetcdf/1.13.0 + hdf5/1.14.4 + /work/02503/edwardsj/vista/modulefiles/nvhpc/24.7.0/openmpi/5.0.5 + parallelio/2.6.3 + esmf/8.7.0b14 + + + + 256M + ON + SUMMARY + + + /scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_nvhpc_24.7 + + + /work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/lib + /work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/include + /scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_gcc_14.2 + /scratch/00422/cazes/pnetcdf_1.13.0_gcc_14.2/ + /work/02503/edwardsj/vista/esmf/v8.6.2b00_gnu_openmpi/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk + /home1/apps/nvidia/Linux_aarch64/24.7/compilers/lib:/scratch/projects/compilers/gcc11/openmpi/5.0.3_cpu/lib:/scratch/projects/compilers/ucx/1.17/lib/ucx:/scratch/projects/compilers/ucx/1.17/lib:/opt/apps/gcc/14.2.0/lib64:/opt/apps/gcc/14.2.0/lib + + diff --git a/machines/vista/nvhpc_vista.cmake b/machines/vista/nvhpc_vista.cmake new file mode 100644 index 0000000..d61b047 --- /dev/null +++ b/machines/vista/nvhpc_vista.cmake @@ -0,0 +1,5 @@ +set(SUPPORTS_CXX "TRUE") +string(APPEND SLIBS " -L$ENV{NETCDFHOME}/lib -lnetcdff -lnetcdf ") +if (MPILIB STREQUAL openmpi) + string(APPEND SLIBS " -L$ENV{PNETCDFHOME}/lib -lpnetcdf ") +endif() diff --git a/machines/vista/vista.cmake b/machines/vista/vista.cmake new file mode 100644 index 0000000..f8a605a --- /dev/null +++ b/machines/vista/vista.cmake @@ -0,0 +1,3 @@ +string(APPEND LDFLAGS " -Wl,-rpath,/scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_nvhpc_24.7/lib") +string(APPEND LDFLAGS " -Wl,-rpath,/scratch/00422/cazes/pnetcdf_1.13.0_nvhpc/lib") +set(LAPACK_LIBDIR "/home1/apps/nvidia/Linux_aarch64/24.7/compilers/lib")