lanl · rbberger · Oct 1, 2024 · Sep 23, 2024 · Sep 23, 2024
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,49 +1,42 @@
 stages:
+ - static_analysis
  - build_n_test
 
-default:
- interruptible: true
-
 variables:
- ENABLED_CLUSTERS: "darwin"
- GIT_SUBMODULE_STRATEGY: normal
+ PROJECT_NAME: ports-of-call
+ ENABLE_CDASH: false
 
-.ascgit_job:
- id_tokens:
- SITE_ID_TOKEN:
- aud: https://asc-git.lanl.gov
+include:
+ - .gitlab/common.yml
 
-.darwin_job:
- rules:
- - if: $ENABLED_CLUSTERS =~ /darwin/
- variables:
- CLUSTER: darwin
- SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\""
- tags:
- - darwin-slurm-shared
-
-.build_and_test:
- stage: build_n_test
- script:
- - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}
- artifacts:
- expire_in: 2 weeks
- paths:
- - ${CI_PROJECT_DIR}/build/tests.xml
- reports:
- junit: ${CI_PROJECT_DIR}/build/tests.xml
 
 ########
 # Jobs #
 ########
 
 openmpi_gcc:
- extends: [.ascgit_job, .darwin_job, .build_and_test]
+ extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
  variables:
  SPACK_ENV_NAME: openmpi-gcc
 
+openmpi_cuda_gcc_volta:
+ extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
+ variables:
+ SPACK_ENV_NAME: openmpi-cuda-gcc-volta
+ SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86 -C cpu_family:haswell --time=02:00:00"
+
 openmpi_cuda_gcc_ampere:
  extends: [.ascgit_job, .darwin_job, .build_and_test]
  variables:
  SPACK_ENV_NAME: openmpi-cuda-gcc-ampere
  SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
+
+rzvernal_craympich_rocm_mi250_gcc:
+ extends: [.ascgit_job, .rzadams_job, .rzvernal_regular_job, .build_and_test]
+ variables:
+ SPACK_ENV_NAME: craympich-rocm-gfx90a-gcc
+
+rzadams_craympich_rocm_mi300_gcc:
+ extends: [.ascgit_job, .rzadams_job, .rzadams_regular_job, .build_and_test]
+ variables:
+ SPACK_ENV_NAME: craympich-rocm-gfx942-gcc
diff --git a/.gitlab/build_and_test.cmake b/.gitlab/build_and_test.cmake
@@ -0,0 +1,75 @@
+set(CTEST_SOURCE_DIRECTORY "$ENV{SOURCE_DIR}")
+set(CTEST_BINARY_DIRECTORY "$ENV{BUILD_DIR}")
+set(CTEST_PROJECT_NAME "$ENV{PROJECT_NAME}")
+
+
+set(CTEST_SITE "$ENV{SYSTEM_NAME}")
+set(CTEST_BUILD_NAME "$ENV{SPACK_ENV_NAME}")
+
+set(CTEST_SUBMIT_URL "$ENV{CDASH_SERVER_URL}/submit.php?project=${CTEST_PROJECT_NAME}")
+set(CTEST_CURL_OPTIONS CURLOPT_SSL_VERIFYPEER_OFF CURLOPT_SSL_VERIFYHOST_OFF)
+set(CTEST_NIGHTLY_START_TIME "22:00:00 MDT")
+set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1024000)
+set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1024000)
+
+set(CTEST_UPDATE_COMMAND "git")
+set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "Skipping git update (no-op).")
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo")
+
+set(CTEST_MODE "$ENV{CTEST_MODE}")
+
+set(CTEST_OUTPUT_ON_FAILURE ON)
+set(CTEST_USE_LAUNCHERS TRUE)
+
+cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES)
+
+if(${NUM_PHYSICAL_CORES} EQUAL 1)
+ # workaround for GraceHopper
+ include(ProcessorCount)
+ ProcessorCount(NUM_PHYSICAL_CORES)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+ ctest_start(${CTEST_MODE})
+else()
+ ctest_start(${CTEST_MODE} APPEND)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+ ctest_update(SOURCE "${CTEST_SOURCE_DIRECTORY}")
+ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE configure_error)
+
+ if(configure_error)
+ if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+ ctest_submit()
+ endif()
+ message(FATAL_ERROR "configure error")
+ endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Build)
+ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" PARALLEL_LEVEL ${NUM_PHYSICAL_CORES} RETURN_VALUE build_error)
+
+ if(build_error)
+ if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+ ctest_submit()
+ endif()
+ message(FATAL_ERROR "build error")
+ endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Test)
+ ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" OUTPUT_JUNIT tests.xml RETURN_VALUE test_error)
+
+ if(test_error)
+ if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+ ctest_submit()
+ endif()
+ message(FATAL_ERROR "test error")
+ endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Submit)
+ ctest_submit()
+endif()
diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh
@@ -1,12 +1,33 @@
 #!/bin/bash
-PROJECT_NAME=ports-of-call
-PROJECT_DEFAULT_BRANCH=main
-PROJECT_GROUP=oss
-BUILD_DIR=${BUILD_DIR:-build}
-SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
-PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/current}
+###############################################################################
+export PROJECT_NAME=ports-of-call
+export PROJECT_DEFAULT_BRANCH=main
+export PROJECT_TYPE=oss
+export PROJECT_GROUP=oss
+###############################################################################
+
+export BUILD_DIR=${BUILD_DIR:-build}
+export SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
+TMPDIR=${TMPDIR:-/tmp/xcap/${PROJECT_GROUP}/${PROJECT_NAME}}
 CI_SPACK_ENV=${CI_SPACK_ENV:-$TMPDIR/$USER-ci-env}
-UNTIL=${UNTIL:-install}
+export CTEST_MODE=${CTEST_MODE:-Continuous}
+SUBMIT_TO_CDASH=${SUBMIT_TO_CDASH:-false}
+BUILD_WITH_CTEST=${BUILD_WITH_CTEST:-${SUBMIT_TO_CDASH}}
+SUBMIT_ON_ERROR=${SUBMIT_ON_ERROR:-${SUBMIT_TO_CDASH}}
+SHOW_HELP_MESSAGE=${SHOW_HELP_MESSAGE:-true}
+
+if ${SUBMIT_TO_CDASH}; then
+ UNTIL=${UNTIL:-submit}
+else
+ UNTIL=${UNTIL:-install}
+fi
+
+if ${SUBMIT_ON_ERROR}; then
+ REPORT_ERRORS="ReportErrors"
+else
+ REPORT_ERRORS=""
+fi
+PROJECT_SPACK_ENV_VERSION=${PROJECT_SPACK_ENV_VERSION:-current}
 
 # colors
 COLOR_BLUE='\033[1;34m'
@@ -47,7 +68,7 @@ print_usage() {
  echo " environment name of the Spack environment"
  echo
  echo "options:"
- echo " -u {spack,env,cmake,build,test,install}, --until {spack,env,cmake,build,test,install}"
+ echo " -u {spack,env,cmake,build,test,install,submit}, --until {spack,env,cmake,build,test,install,submit}"
  echo " run script until the given phase"
  echo " -h, --help show this help message and exit"
 }
@@ -67,11 +88,34 @@ fi
 
 if ! $VALID_CMD; then
  print_usage
+ false
  return
 fi
 
-SYSTEM_NAME=$1
-SPACK_ENV_NAME=$2
+export SYSTEM_NAME=$1
+export SPACK_ENV_NAME=$2
+
+if [[ "$SYSTEM_NAME" == "darwin" || "$SYSTEM_NAME" == "rocinante" || "$SYSTEM_NAME" == "venado" ]]; then
+ PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}}
+elif [[ "$SYSTEM_NAME" == "rzadams" || "${SYSTEM_NAME}" == "rzansel" || "$SYSTEM_NAME" == "rzvernal" ]]; then
+ PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/workspace/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}}
+else
+ echo "Unkown system '${SYSTEM_NAME}'"
+ false
+ return
+fi
+
+if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] && [[ -z "${SPACK_ENV_SPEC}" ]]; then
+ echo "Spack environment 'custom-spec' requires SPACK_ENV_SPEC environment variable to be set!"
+ false
+ return
+fi
+
+if [[ "${SPACK_ENV_NAME}" == "custom-file" ]] && [[ -z "${SPACK_ENV_FILE}" ]]; then
+ echo "Spack environment 'custom-file' requires SPACK_ENV_FILE environment variable to be set!"
+ false
+ return
+fi
 
 ###############################################################################
 # Generic steps
@@ -83,6 +127,7 @@ SPACK_ENV_NAME=$2
 prepare_spack() {
  section start "prepare_spack[collapsed=true]" "Prepare Spack"
  umask 0007
+ mkdir -p $TMPDIR
  source $PROJECT_SPACK_ENV/replicate.sh $CI_SPACK_ENV
  section end "prepare_spack"
 }
@@ -93,12 +138,27 @@ prepare_spack() {
 prepare_env() {
  section start "prepare_env[collapsed=true]" "Create environment"
  echo "Activating ${SPACK_ENV_NAME} environment on ${SYSTEM_NAME}"
+
+ if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then
+ SPACK_ENV_FILE=${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/custom/spack.yaml
+ fi
+
+ if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] || [[ "${SPACK_ENV_NAME}" == "custom-file" ]]; then
+ mkdir -p ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
+ cp ${SPACK_ENV_FILE} ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}/spack.yaml
+ fi
+
  source ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/activate.sh ${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
  if [ -d ${SOURCE_DIR}/spack-repo ]; then
  spack repo add ${SOURCE_DIR}/spack-repo
  fi
 
  spack develop -b ${BUILD_DIR} -p ${SOURCE_DIR} --no-clone ${PROJECT_NAME}@${PROJECT_DEFAULT_BRANCH}
+
+ if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then
+ spack add ${SPACK_ENV_SPEC}
+ fi
+
  spack install --include-build-deps --only dependencies -j $(nproc)
  section end "prepare_env"
 }
@@ -122,7 +182,11 @@ cmake_build() {
  (
  source ${BUILD_ENV}
  cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/${BUILD_DIR}/install $@ ${BUILD_DIR}
- cmake --build ${BUILD_DIR} --parallel
+ if ${BUILD_WITH_CTEST}; then
+ ctest -VV -S .gitlab/build_and_test.cmake,Configure,Build,$REPORT_ERRORS
+ else
+ cmake --build ${BUILD_DIR} --parallel
+ fi
  )
  section end "cmake_build"
 }
@@ -135,11 +199,27 @@ cmake_test() {
  (
  source ${BUILD_ENV}
  export CTEST_OUTPUT_ON_FAILURE=1
- ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@
+ if ${BUILD_WITH_CTEST}; then
+ ctest -V -S .gitlab/build_and_test.cmake,Test,$REPORT_ERRORS
+ else
+ ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@
+ fi
  )
  section end "testing"
 }
 
+#------------------------------------------------------------------------------
+# Submit results to CDash
+#------------------------------------------------------------------------------
+cmake_submit() {
+ section start "submit[collapsed=false]" "Submit to CDash"
+ (
+ source ${BUILD_ENV}
+ ctest -V -S .gitlab/build_and_test.cmake,Submit
+ )
+ section end "submit"
+}
+
 #------------------------------------------------------------------------------
 # Install project
 #------------------------------------------------------------------------------
@@ -162,23 +242,37 @@ activate_build_env() {
 ###############################################################################
 # Run script
 ###############################################################################
-echo "Running on $(hostname)"
-
-if [ ${CI} ]; then
- echo " "
- echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
- echo " "
- echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
- echo " "
- echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
- echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}"
- echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
- echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh --until ${UNTIL} ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
- echo " "
- echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
- echo " "
- echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
- echo " "
+if ${SHOW_HELP_MESSAGE}; then
+ echo "Running on $(hostname)"
+
+ if [ ${CI} ]; then
+ echo " "
+ echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+ echo " "
+ echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
+ echo " "
+ echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
+ echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}"
+ if [[ ! -z "${LLNL_FLUX_SCHEDULER_PARAMETERS}" ]]; then
+ echo -e "${COLOR_BLUE}flux alloc ${LLNL_FLUX_SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+ elif [[ ! -z "${LLNL_LSF_SCHEDULER_PARAMETERS}" ]]; then
+ echo -e "${COLOR_BLUE}bsub -I ${LLNL_LSF_SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+ else
+ echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+ fi
+ if [[ ! -z "${SPACK_ENV_SPEC}" ]]; then
+ echo -e "${COLOR_BLUE}export SPACK_ENV_SPEC='${SPACK_ENV_SPEC}'${COLOR_PLAIN}"
+ fi
+ if [[ ! -z "${SPACK_ENV_FILE}" ]]; then
+ echo -e "${COLOR_BLUE}export SPACK_ENV_FILE='${SPACK_ENV_FILE}'${COLOR_PLAIN}"
+ fi
+ echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
+ echo " "
+ echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
+ echo " "
+ echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+ echo " "
+ fi
 fi
 
 prepare_spack
@@ -198,3 +292,7 @@ if [[ "${UNTIL}" == "test" ]]; then return; fi
 
 cmake_install
 if [[ "${UNTIL}" == "install" ]]; then return; fi
+
+if ${SUBMIT_TO_CDASH}; then
+ cmake_submit
+fi