diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ede93f582..d9b152139 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,49 +1,62 @@ stages: - build_n_test -default: - interruptible: true - variables: - ENABLED_CLUSTERS: "darwin" - GIT_SUBMODULE_STRATEGY: normal + PROJECT_NAME: spiner + ENABLE_CDASH: false -.ascgit_job: - id_tokens: - SITE_ID_TOKEN: - aud: https://asc-git.lanl.gov +include: + - .gitlab/common.yml -.darwin_job: - rules: - - if: $ENABLED_CLUSTERS =~ /darwin/ - variables: - CLUSTER: darwin - SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\"" - tags: - - darwin-slurm-shared -.build_and_test: +######## +# Jobs # +######## + +prereq_offline_deps: stage: build_n_test + extends: [.ascgit_job] + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "push" + needs: [] script: - - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME} + - mkdir extern + - git clone --bare https://gitlab-ci-token:${CI_JOB_TOKEN}@re-git.lanl.gov/xcap/oss/ports-of-call.git extern/ports-of-call.git artifacts: - expire_in: 2 weeks + expire_in: 2 days paths: - - ${CI_PROJECT_DIR}/build/tests.xml - reports: - junit: ${CI_PROJECT_DIR}/build/tests.xml - -######## -# Jobs # -######## + - extern openmpi_gcc: - extends: [.ascgit_job, .darwin_job, .build_and_test] + extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test] variables: SPACK_ENV_NAME: openmpi-gcc +openmpi_cuda_gcc_volta: + extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test] + variables: + SPACK_ENV_NAME: openmpi-cuda-gcc-volta + SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86 -C cpu_family:haswell --time=02:00:00" + openmpi_cuda_gcc_ampere: - extends: [.ascgit_job, .darwin_job, .build_and_test] + extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test] variables: SPACK_ENV_NAME: openmpi-cuda-gcc-ampere SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere" + +rzvernal_craympich_rocm_mi250_gcc: + extends: [.ascgit_job, .rzadams_job, .rzvernal_regular_job, .build_and_test] + needs: + - prereq_offline_deps + variables: + SPACK_ENV_NAME: craympich-rocm-gfx90a-gcc + +rzadams_craympich_rocm_mi300_gcc: + extends: [.ascgit_job, .rzadams_job, .rzadams_regular_job, .build_and_test] + needs: + - prereq_offline_deps + variables: + SPACK_ENV_NAME: craympich-rocm-gfx942-gcc diff --git a/.gitlab/build_and_test.cmake b/.gitlab/build_and_test.cmake new file mode 100644 index 000000000..a79d508f9 --- /dev/null +++ b/.gitlab/build_and_test.cmake @@ -0,0 +1,75 @@ +set(CTEST_SOURCE_DIRECTORY "$ENV{SOURCE_DIR}") +set(CTEST_BINARY_DIRECTORY "$ENV{BUILD_DIR}") +set(CTEST_PROJECT_NAME "$ENV{PROJECT_NAME}") + + +set(CTEST_SITE "$ENV{SYSTEM_NAME}") +set(CTEST_BUILD_NAME "$ENV{SPACK_ENV_NAME}") + +set(CTEST_SUBMIT_URL "$ENV{CDASH_SERVER_URL}/submit.php?project=${CTEST_PROJECT_NAME}") +set(CTEST_CURL_OPTIONS CURLOPT_SSL_VERIFYPEER_OFF CURLOPT_SSL_VERIFYHOST_OFF) +set(CTEST_NIGHTLY_START_TIME "22:00:00 MDT") +set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1024000) +set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1024000) + +set(CTEST_UPDATE_COMMAND "git") +set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "Skipping git update (no-op).") +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo") + +set(CTEST_MODE "$ENV{CTEST_MODE}") + +set(CTEST_OUTPUT_ON_FAILURE ON) +set(CTEST_USE_LAUNCHERS TRUE) + +cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES) + +if(${NUM_PHYSICAL_CORES} EQUAL 1) + # workaround for GraceHopper + include(ProcessorCount) + ProcessorCount(NUM_PHYSICAL_CORES) +endif() + +if(${CTEST_SCRIPT_ARG} MATCHES Configure) + ctest_start(${CTEST_MODE}) +else() + ctest_start(${CTEST_MODE} APPEND) +endif() + +if(${CTEST_SCRIPT_ARG} MATCHES Configure) + ctest_update(SOURCE "${CTEST_SOURCE_DIRECTORY}") + ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE configure_error) + + if(configure_error) + if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors) + ctest_submit() + endif() + message(FATAL_ERROR "configure error") + endif() +endif() + +if(${CTEST_SCRIPT_ARG} MATCHES Build) + ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" PARALLEL_LEVEL ${NUM_PHYSICAL_CORES} RETURN_VALUE build_error) + + if(build_error) + if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors) + ctest_submit() + endif() + message(FATAL_ERROR "build error") + endif() +endif() + +if(${CTEST_SCRIPT_ARG} MATCHES Test) + ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" OUTPUT_JUNIT tests.xml RETURN_VALUE test_error) + + if(test_error) + if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors) + ctest_submit() + endif() + message(FATAL_ERROR "test error") + endif() +endif() + +if(${CTEST_SCRIPT_ARG} MATCHES Submit) + ctest_submit() +endif() diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index 07b2cf150..d9d435ef5 100644 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -1,12 +1,33 @@ #!/bin/bash -PROJECT_NAME=spiner -PROJECT_DEFAULT_BRANCH=main -PROJECT_GROUP=oss -BUILD_DIR=${BUILD_DIR:-build} -SOURCE_DIR=${CI_PROJECT_DIR:-$PWD} -PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/current} +############################################################################### +export PROJECT_NAME=spiner +export PROJECT_DEFAULT_BRANCH=main +export PROJECT_TYPE=oss +export PROJECT_GROUP=oss +############################################################################### + +export BUILD_DIR=${BUILD_DIR:-build} +export SOURCE_DIR=${CI_PROJECT_DIR:-$PWD} +TMPDIR=${TMPDIR:-/tmp/xcap/${PROJECT_GROUP}/${PROJECT_NAME}} CI_SPACK_ENV=${CI_SPACK_ENV:-$TMPDIR/$USER-ci-env} -UNTIL=${UNTIL:-install} +export CTEST_MODE=${CTEST_MODE:-Continuous} +SUBMIT_TO_CDASH=${SUBMIT_TO_CDASH:-false} +BUILD_WITH_CTEST=${BUILD_WITH_CTEST:-${SUBMIT_TO_CDASH}} +SUBMIT_ON_ERROR=${SUBMIT_ON_ERROR:-${SUBMIT_TO_CDASH}} +SHOW_HELP_MESSAGE=${SHOW_HELP_MESSAGE:-true} + +if ${SUBMIT_TO_CDASH}; then + UNTIL=${UNTIL:-submit} +else + UNTIL=${UNTIL:-install} +fi + +if ${SUBMIT_ON_ERROR}; then + REPORT_ERRORS="ReportErrors" +else + REPORT_ERRORS="" +fi +PROJECT_SPACK_ENV_VERSION=${PROJECT_SPACK_ENV_VERSION:-current} # colors COLOR_BLUE='\033[1;34m' @@ -47,7 +68,7 @@ print_usage() { echo " environment name of the Spack environment" echo echo "options:" - echo " -u {spack,env,cmake,build,test,install}, --until {spack,env,cmake,build,test,install}" + echo " -u {spack,env,cmake,build,test,install,submit}, --until {spack,env,cmake,build,test,install,submit}" echo " run script until the given phase" echo " -h, --help show this help message and exit" } @@ -67,11 +88,34 @@ fi if ! $VALID_CMD; then print_usage + false return fi -SYSTEM_NAME=$1 -SPACK_ENV_NAME=$2 +export SYSTEM_NAME=$1 +export SPACK_ENV_NAME=$2 + +if [[ "$SYSTEM_NAME" == "darwin" || "$SYSTEM_NAME" == "rocinante" || "$SYSTEM_NAME" == "venado" ]]; then + PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}} +elif [[ "$SYSTEM_NAME" == "rzadams" || "${SYSTEM_NAME}" == "rzansel" || "$SYSTEM_NAME" == "rzvernal" ]]; then + PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/workspace/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}} +else + echo "Unkown system '${SYSTEM_NAME}'" + false + return +fi + +if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] && [[ -z "${SPACK_ENV_SPEC}" ]]; then + echo "Spack environment 'custom-spec' requires SPACK_ENV_SPEC environment variable to be set!" + false + return +fi + +if [[ "${SPACK_ENV_NAME}" == "custom-file" ]] && [[ -z "${SPACK_ENV_FILE}" ]]; then + echo "Spack environment 'custom-file' requires SPACK_ENV_FILE environment variable to be set!" + false + return +fi ############################################################################### # Generic steps @@ -83,6 +127,7 @@ SPACK_ENV_NAME=$2 prepare_spack() { section start "prepare_spack[collapsed=true]" "Prepare Spack" umask 0007 + mkdir -p $TMPDIR source $PROJECT_SPACK_ENV/replicate.sh $CI_SPACK_ENV section end "prepare_spack" } @@ -93,12 +138,31 @@ prepare_spack() { prepare_env() { section start "prepare_env[collapsed=true]" "Create environment" echo "Activating ${SPACK_ENV_NAME} environment on ${SYSTEM_NAME}" + + if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then + SPACK_ENV_FILE=${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/custom/spack.yaml + fi + + if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] || [[ "${SPACK_ENV_NAME}" == "custom-file" ]]; then + mkdir -p ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME} + cp ${SPACK_ENV_FILE} ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}/spack.yaml + fi + source ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/activate.sh ${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME} if [ -d ${SOURCE_DIR}/spack-repo ]; then spack repo add ${SOURCE_DIR}/spack-repo fi + if [ -d ${SOURCE_DIR}/extern/ports-of-call.git ]; then + spack config add "packages:ports-of-call:package_attributes:git:'file://${SOURCE_DIR}/extern/ports-of-call.git'" + fi + spack develop -b ${BUILD_DIR} -p ${SOURCE_DIR} --no-clone ${PROJECT_NAME}@${PROJECT_DEFAULT_BRANCH} + + if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then + spack add ${SPACK_ENV_SPEC} + fi + spack install --include-build-deps --only dependencies -j $(nproc) section end "prepare_env" } @@ -122,7 +186,11 @@ cmake_build() { ( source ${BUILD_ENV} cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/${BUILD_DIR}/install $@ ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel + if ${BUILD_WITH_CTEST}; then + ctest -VV -S .gitlab/build_and_test.cmake,Configure,Build,$REPORT_ERRORS + else + cmake --build ${BUILD_DIR} --parallel + fi ) section end "cmake_build" } @@ -135,11 +203,27 @@ cmake_test() { ( source ${BUILD_ENV} export CTEST_OUTPUT_ON_FAILURE=1 - ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@ + if ${BUILD_WITH_CTEST}; then + ctest -V -S .gitlab/build_and_test.cmake,Test,$REPORT_ERRORS + else + ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@ + fi ) section end "testing" } +#------------------------------------------------------------------------------ +# Submit results to CDash +#------------------------------------------------------------------------------ +cmake_submit() { + section start "submit[collapsed=false]" "Submit to CDash" + ( + source ${BUILD_ENV} + ctest -V -S .gitlab/build_and_test.cmake,Submit + ) + section end "submit" +} + #------------------------------------------------------------------------------ # Install project #------------------------------------------------------------------------------ @@ -162,23 +246,40 @@ activate_build_env() { ############################################################################### # Run script ############################################################################### -echo "Running on $(hostname)" - -if [ ${CI} ]; then - echo " " - echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}" - echo " " - echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}" - echo " " - echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}" - echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}" - echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}" - echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh --until ${UNTIL} ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}" - echo " " - echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}" - echo " " - echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}" - echo " " +if ${SHOW_HELP_MESSAGE}; then + echo "Running on $(hostname)" + + if [ ${CI} ]; then + echo " " + echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}" + echo " " + echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}" + echo " " + echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}" + echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}" + if [ -d ${SOURCE_DIR}/extern/ports-of-call.git ]; then + echo -e "${COLOR_BLUE}git clone --bare ssh://git@re-git.lanl.gov:10022/xcap/oss/ports-of-call.git extern/ports-of-call.git${COLOR_PLAIN}" + fi + if [[ ! -z "${LLNL_FLUX_SCHEDULER_PARAMETERS}" ]]; then + echo -e "${COLOR_BLUE}flux alloc ${LLNL_FLUX_SCHEDULER_PARAMETERS}${COLOR_PLAIN}" + elif [[ ! -z "${LLNL_LSF_SCHEDULER_PARAMETERS}" ]]; then + echo -e "${COLOR_BLUE}bsub -I ${LLNL_LSF_SCHEDULER_PARAMETERS}${COLOR_PLAIN}" + else + echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}" + fi + if [[ ! -z "${SPACK_ENV_SPEC}" ]]; then + echo -e "${COLOR_BLUE}export SPACK_ENV_SPEC='${SPACK_ENV_SPEC}'${COLOR_PLAIN}" + fi + if [[ ! -z "${SPACK_ENV_FILE}" ]]; then + echo -e "${COLOR_BLUE}export SPACK_ENV_FILE='${SPACK_ENV_FILE}'${COLOR_PLAIN}" + fi + echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}" + echo " " + echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}" + echo " " + echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}" + echo " " + fi fi prepare_spack @@ -198,3 +299,7 @@ if [[ "${UNTIL}" == "test" ]]; then return; fi cmake_install if [[ "${UNTIL}" == "install" ]]; then return; fi + +if ${SUBMIT_TO_CDASH}; then + cmake_submit +fi diff --git a/.gitlab/common.yml b/.gitlab/common.yml new file mode 100644 index 000000000..506d28105 --- /dev/null +++ b/.gitlab/common.yml @@ -0,0 +1,239 @@ +default: + interruptible: true + before_script: + - unset CI_MERGE_REQUEST_DESCRIPTION + - unset CI_MERGE_REQUEST_TITLE + +.ascgit_job: + id_tokens: + SITE_ID_TOKEN: + aud: https://asc-git.lanl.gov + +.darwin_job: + allow_failure: true + variables: + CLUSTER: darwin + SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\"" + tags: + - darwin-slurm-shared + +.darwin_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "pipeline" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "push" + +.darwin_regular_mr_only_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "schedule" + when: never + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_TAG + when: never + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "pipeline" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "push" + +.darwin_regular_default_branch_only_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "pipeline" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push" + +.darwin_nightly_job: + variables: + CTEST_MODE: Nightly + rules: + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_MERGE_REQUEST_LABELS =~ /nightly/ + +.rocinante_job: + allow_failure: true + variables: + CLUSTER: rocinante + SCHEDULER_PARAMETERS: "-N 1 -A asc-ci -p ci --reservation ci --time=02:00:00" + tags: + - rocinante + - batch + +.rocinante_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "push" + +.venado_job: + allow_failure: true + variables: + CLUSTER: venado + SCHEDULER_PARAMETERS: "-N 1 -p standard --time=02:00:00" + tags: + - venado + - batch + +.venado_shell_gpu_job: + variables: + CLUSTER: venado + tags: + - venado + - shell + - gpu + +.venado_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "push" + +.rzansel_job: + allow_failure: true + tags: + - rzansel + - batch + variables: + CLUSTER: rzansel + LLNL_LSF_SCHEDULER_PARAMETERS: "--nnodes=1" + +.rzansel_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "push" + +.rzvernal_job: + allow_failure: true + tags: + - rzvernal + - batch + - flux + variables: + CLUSTER: rzvernal + LLNL_FLUX_SCHEDULER_PARAMETERS: "-N=1 --time-limit=2h" + +.rzvernal_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "push" + +.rzadams_job: + allow_failure: true + tags: + - rzadams + - batch + - flux + variables: + CLUSTER: rzadams + LLNL_FLUX_SCHEDULER_PARAMETERS: "-N=1 --time-limit=2h" + +.rzadams_regular_job: + variables: + CTEST_MODE: Continuous + rules: + - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "schedule" + - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "web" + - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "push" + +.build_and_test: + stage: build_n_test + variables: + SUBMIT_TO_CDASH: false + SUBMIT_AFTER: false + CREATE_SUBMIT_ARTIFACT: false + TEST_TARFILE: "${CLUSTER}-${CTEST_MODE}-${SPACK_ENV_NAME}.tar.gz" + script: + - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME} + after_script: + - touch ${TEST_TARFILE} + - | + if ${SUBMIT_AFTER}; then + export SOURCE_DIR=$PWD + export BUILD_DIR=build + export SYSTEM_NAME=${CLUSTER} + ctest -V -S .gitlab/build_and_test.cmake,Submit + elif ${CREATE_SUBMIT_ARTIFACT}; then + rm -f ${TEST_TARFILE} + cd build + tar cvzf ../${TEST_TARFILE} DartConfiguration.tcl Testing + fi + artifacts: + when: always + expire_in: 2 weeks + paths: + - ${CI_PROJECT_DIR}/build/tests.xml + - ${CI_PROJECT_DIR}/${TEST_TARFILE} + reports: + junit: ${CI_PROJECT_DIR}/build/tests.xml + +.build_docs: + stage: static_analysis + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - git fetch --prune --prune-tags + - git fetch --tags + - if [[ ${CI_JOB_NAME%%:*} =~ "pages_check" ]]; then git tag ${CI_JOB_NAME%%:*}; fi + - source .gitlab/build_docs.sh ${CLUSTER} + - rm -rf public + - mv build/doc/html public + - if [[ ${CI_JOB_NAME%%:*} =~ "pages_check" ]]; then git tag -d ${CI_JOB_NAME%%:*}; fi + +.clang_format: + stage: static_analysis + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - source .gitlab/check_format.sh ${CLUSTER} + +.post_results_to_cdash: + stage: build_n_test + id_tokens: + SITE_ID_TOKEN: + aud: https://asc-git.lanl.gov + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - cmake --version + - ls *.tar.gz + - export SOURCE_DIR=$PWD + - export BUILD_DIR=build + - | + for t in $(ls *.tar.gz); do + export SYSTEM_NAME=$(echo $t | cut -d - -f 1) + export CTEST_MODE=$(echo $t | cut -d - -f 2) + FULL_NAME=${t%%.*} + export SPACK_ENV_NAME=${FULL_NAME/${SYSTEM_NAME}-${CTEST_MODE}-/} + mkdir -p ${BUILD_DIR} + cd ${BUILD_DIR} + echo "#############################################################################" + echo "Submitting result for ${CTEST_MODE} with ${SPACK_ENV_NAME} on ${SYSTEM_NAME}" + echo "#############################################################################" + tar xvzf ../$t + cd ${CI_PROJECT_DIR} + ctest -V -S .gitlab/build_and_test.cmake,Submit + rm -rf ${BUILD_DIR} + done diff --git a/spack-repo/packages/ports-of-call/package.py b/spack-repo/packages/ports-of-call/package.py index 38137f37e..04fe02bd3 100644 --- a/spack-repo/packages/ports-of-call/package.py +++ b/spack-repo/packages/ports-of-call/package.py @@ -62,4 +62,9 @@ def cmake_args(self): ] if self.spec.satisfies("@:1.2.0"): args.append(self.define_from_variant("PORTABILITY_STRATEGY", "portability_strategy")) + if self.spec.satisfies("test_portability_strategy=Kokkos ^kokkos+rocm"): + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc)) + args.append(self.define("CMAKE_C_COMPILER", self.spec["hip"].hipcc)) + if self.spec.satisfies("test_portability_strategy=Kokkos ^kokkos+cuda"): + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["kokkos"].kokkos_cxx)) return args diff --git a/spack-repo/packages/spiner/package.py b/spack-repo/packages/spiner/package.py index 74feaf2b3..e7275f17f 100644 --- a/spack-repo/packages/spiner/package.py +++ b/spack-repo/packages/spiner/package.py @@ -95,4 +95,9 @@ def cmake_args(self): args.append( self.define("CMAKE_CUDA_ARCHITECTURES", self.spec["kokkos"].variants["cuda_arch"].value) ) + if self.spec.satisfies("^kokkos+rocm"): + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc)) + args.append(self.define("CMAKE_C_COMPILER", self.spec["hip"].hipcc)) + if self.spec.satisfies("^kokkos+cuda"): + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["kokkos"].kokkos_cxx)) return args