diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 219a1e5afe6af1c587e4cc569314cfb4ace73db3..4891f2d0cfe661a7424e83d7588118992b3010ef 100755 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -27,6 +27,10 @@ pipeline { description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.') } + environment { + CONTAINER_STYLE = "visionary" + } + stages { stage('Container Build') { agent { label 'conviz1||conviz2' } @@ -43,7 +47,7 @@ pipeline { VISIONARY_GCC = "gcc@${VISIONARY_GCC_VERSION}" TMPDIR = "/tmp/${env.NODE_NAME}" JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX", - returnStdout: true).trim() + returnStdout: true).trim() BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment } stages { @@ -98,7 +102,7 @@ pipeline { } stage('Create visionary recipe') { steps { - sh ".ci/create_visionary_recipe.sh" + sh ".ci/visionary_create_recipe.sh" } } stage('Build sandbox') { diff --git a/.ci/Jenkinsfile_asic b/.ci/Jenkinsfile_asic index 59f5ec32550eb693bcd39ba15dd407cf462bc898..b4d58ec9ab21ed2622ff5c933ca8c9bb64b36884 100755 --- a/.ci/Jenkinsfile_asic +++ b/.ci/Jenkinsfile_asic @@ -16,6 +16,10 @@ pipeline { skipDefaultCheckout() } + environment { + CONTAINER_STYLE = "asic" + } + stages { stage('Container Build') { agent { label 'conviz1||conviz2' } @@ -24,6 +28,7 @@ pipeline { TMPDIR = "/tmp/${env.NODE_NAME}" JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX", returnStdout: true).trim() + BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment } stages { stage('Pre-build Cleanup') { @@ -46,19 +51,19 @@ pipeline { } stage('Build sandbox') { steps { - sh ".ci/asic_build_sandbox.sh" + sh ".ci/build_sandbox.sh" } } stage('Build container image') { steps { - sh ".ci/asic_build_image.sh" + sh ".ci/build_image.sh" } } stage('Update build cache and export container') { steps { script { // we only want the container name, tail everything else - CONTAINER_NAME = sh(script: ".ci/asic_deploy_container.sh | tail -n 1", returnStdout: true).trim() + CONTAINER_IMAGE = sh(script: ".ci/deploy_container.sh | tail -n 1", returnStdout: true).trim() } } } diff --git a/.ci/asic_build_image.sh b/.ci/asic_build_image.sh deleted file mode 100755 index d0144a2cfdd099bb8826d429dfdb1a1549c085f3..0000000000000000000000000000000000000000 --- a/.ci/asic_build_image.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -euo pipefail -shopt -s inherit_errexit - -if (( $(find sandboxes -mindepth 1 -maxdepth 1 | wc -l) > 1 )); then - echo "More than one sandbox found, this should never happen!" >&2 - exit 1 -fi - -TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)" - -# create image file -IMAGE_NAME=singularity_asic_temp.img - -sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}" - -sudo chown -R vis_jenkins singularity_asic_*.img - -if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then - # allow spack user to execute image - chmod +rx singularity_asic_*.img -fi diff --git a/.ci/asic_build_sandbox.sh b/.ci/asic_build_sandbox.sh deleted file mode 100755 index 2c427712b2c708288b81369463f2d9d057c33aa8..0000000000000000000000000000000000000000 --- a/.ci/asic_build_sandbox.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -euo pipefail -shopt -s inherit_errexit - -# set generic locale for building -export LANG=C.UTF-8 -export LC_ALL=C.UTF-8 - -unset LC_CTYPE -unset LC_COLLATE -unset LC_MONETARY -unset LC_NUMERIC -unset LC_TIME -unset LC_MESSAGES - -echo "creating asic_recipe.def" >&2 -SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")" -"${SOURCE_DIR}/asic_create_recipe.sh" - -echo "created asic_recipe.def" >&2 -cat "${WORKSPACE}/asic_recipe.def" - -# build the container (using scripts from above) -export http_proxy=http://proxy.kip.uni-heidelberg.de:8080 -export https_proxy=http://proxy.kip.uni-heidelberg.de:8080 - -TARGET_FOLDER="sandboxes/asic" - -# Do not change: special sudo permit for jenkins user... -sudo rm -rf sandboxes/ - -mkdir sandboxes - -# Do not change: special sudo permit for jenkins user... -sudo -E singularity build --sandbox "${TARGET_FOLDER}" asic_recipe.def diff --git a/.ci/asic_deploy_container.sh b/.ci/asic_deploy_container.sh deleted file mode 100755 index 735599f87025804fc7e69adb8f80e888dba08f68..0000000000000000000000000000000000000000 --- a/.ci/asic_deploy_container.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -euo pipefail -shopt -s inherit_errexit - -# NOTE: stdout of this script is parsed within the Jenkinsfile. -# Think twice before adding any output! - -SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")" -source "${SOURCE_DIR}/asic_dummy_variables.sh" -source "${SOURCE_DIR}/commons.sh" - -INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}" -FALLBACK_DIR="${HOME}/container_mount_full" -IMAGE_NAME="singularity_asic_temp.img" -DATE=$(date --iso) - -get_container_name() -{ - local local_num="$1" - if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then - echo -n "${INSTALL_DIR}/asic_$(get_change_name)_${DATE}_${local_num}.img" - else - echo -n "${INSTALL_DIR}/asic_${DATE}_${local_num}.img" - fi -} - -# find unused image name -num=1 -while [[ -e "$(get_container_name ${num})" ]] ; do - (( num++ )) -done - -CONTAINER_NAME="$(get_container_name ${num})" -# this must be the only output for Jenkins to pick up the container path -# in order to trigger the downstream software builds in the correct container -echo $CONTAINER_NAME - -# copy to target -cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || ( - echo "Error: Copy failed because the mount point is full, saving container image to fallback location.." >&2 - cp -v "${IMAGE_NAME}" "${FALLBACK_DIR}/$(basename "${CONTAINER_NAME}")" >&2 - exit 1 -) - -if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then - ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/asic_latest - - # Announce new container in "Building & Deployment" channel - # Since the output of this script is used in other parts, we have to hide curl's output - export http_proxy=http://proxy.kip.uni-heidelberg.de:8080 - export https_proxy=http://proxy.kip.uni-heidelberg.de:8080 - - curl -i -X POST -H 'Content-Type: application/json' \ - -d "{\"text\": \"@channel New stable ASIC container built: \`${CONTAINER_NAME}\`\"}" \ - https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null -fi - -# delete temporary image -rm "${IMAGE_NAME}" - diff --git a/.ci/bootstrap_spack.sh b/.ci/bootstrap_spack.sh index d349008d615d2e4d4ad4fa690a50b0266248e2ec..4774f0153af8295ea8093f2456c626f2546bbf9a 100755 --- a/.ci/bootstrap_spack.sh +++ b/.ci/bootstrap_spack.sh @@ -58,23 +58,25 @@ echo "" exit 1 fi -# check if it can be specialized -spec_compiler="${VISIONARY_GCC}" -install_from_buildcache "${spec_compiler}" - -# remember system compiler versions (to be removed later) -system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)" - -# upgrade to newer gcc -echo "INSTALL NEW GCC" -set -x -${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}" - -# remove system compilers from spack to avoid conflicting concretization -echo "$(${MY_SPACK_BIN} compiler list)" -for system_compiler in ${system_compilers}; do - ${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}" -done - -# add fresh compiler to spack -${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-* +if [ "${CONTAINER_STYLE}" = "visionary" ]; then + # check if it can be specialized + spec_compiler="${VISIONARY_GCC}" + install_from_buildcache "${spec_compiler}" + + # remember system compiler versions (to be removed later) + system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)" + + # upgrade to newer gcc + echo "INSTALL NEW GCC" + set -x + ${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}" + + # remove system compilers from spack to avoid conflicting concretization + echo "$(${MY_SPACK_BIN} compiler list)" + for system_compiler in ${system_compilers}; do + ${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}" + done + + # add fresh compiler to spack + ${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-* +fi diff --git a/.ci/build_image.sh b/.ci/build_image.sh index 020161a68e3d1c433da25c791fc5319ff9adf2f2..6fcac0614e89c87935054e56c12c6b33b352383c 100755 --- a/.ci/build_image.sh +++ b/.ci/build_image.sh @@ -11,7 +11,7 @@ fi TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)" # create image file -IMAGE_NAME=singularity_spack_temp.img +IMAGE_NAME=singularity_${CONTAINER_STYLE}_temp.img # We want the spack folder to be available inside the container image # -> it needs to be bind mounted to the sandbox folder @@ -22,9 +22,9 @@ sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}" # umount spack folder afterwards sudo umount "${TARGET_FOLDER}/opt/spack" -sudo chown -R vis_jenkins singularity_spack_*.img +sudo chown -R vis_jenkins singularity_${CONTAINER_STYLE}_*.img if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then # allow spack user to execute image - chmod +rx singularity_spack_*.img + chmod +rx singularity_${CONTAINER_STYLE}_*.img fi diff --git a/.ci/build_sandbox.sh b/.ci/build_sandbox.sh index e32af6d69bf6fdc1fda238843145e940c3b57948..299bf95a42472a492af128be2d5d065e8c9d7801 100755 --- a/.ci/build_sandbox.sh +++ b/.ci/build_sandbox.sh @@ -14,12 +14,12 @@ unset LC_NUMERIC unset LC_TIME unset LC_MESSAGES -echo "creating visionary_recipe.def" >&2 +echo "creating ${CONTAINER_STYLE}_recipe.def" >&2 SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")" -"${SOURCE_DIR}/create_visionary_recipe.sh" +"${SOURCE_DIR}/${CONTAINER_STYLE}_create_recipe.sh" -echo "created visionary_recipe.def" >&2 -cat "${WORKSPACE}/visionary_recipe.def" +echo "created ${CONTAINER_STYLE}_recipe.def" >&2 +cat "${WORKSPACE}/${CONTAINER_STYLE}_recipe.def" # create some jenkins-owned temp folder for spack build if [ ! -d "${JOB_TMP_SPACK}" ]; then @@ -34,7 +34,7 @@ chmod 777 "${JOB_TMP_SPACK}" export http_proxy=http://proxy.kip.uni-heidelberg.de:8080 export https_proxy=http://proxy.kip.uni-heidelberg.de:8080 -TARGET_FOLDER="sandboxes/buster_spack" +TARGET_FOLDER="sandboxes/${CONTAINER_STYLE}" # Do not change: special sudo permit for jenkins user... sudo rm -rf sandboxes/ @@ -42,4 +42,4 @@ sudo rm -rf sandboxes/ mkdir sandboxes # Do not change: special sudo permit for jenkins user... -sudo -E singularity build --sandbox "${TARGET_FOLDER}" visionary_recipe.def | tee out_singularity_build_visionary_recipe.txt +sudo -E singularity build --sandbox "${TARGET_FOLDER}" ${CONTAINER_STYLE}_recipe.def | tee out_singularity_build_${CONTAINER_STYLE}_recipe.txt diff --git a/.ci/complete_spack_install_routine_called_in_post_as_root.sh b/.ci/complete_spack_install_routine_called_in_post_as_root.sh index 9aca0cb6054af5575b85a9bec765d4bbd29a85b4..9f9426b7c5b4f6cf11fa94a050ce5bc78c36e028 100755 --- a/.ci/complete_spack_install_routine_called_in_post_as_root.sh +++ b/.ci/complete_spack_install_routine_called_in_post_as_root.sh @@ -11,16 +11,16 @@ shopt -s inherit_errexit 2>/dev/null || true SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")" source "${SOURCE_DIR}/commons.sh" -"${SPACK_INSTALL_SCRIPTS}/prepare_spack_as_root.sh" +"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_prepare_spack_as_root.sh" sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/bootstrap_spack.sh" -sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_visionary_spack.sh" && \ +sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_spack.sh" && \ sudo -Eu spack ${MY_SPACK_BIN} compiler add --scope site /usr/bin && \ sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings.sh" && \ # remove temporary cache folder rm -rfv /opt/spack/.spack -"${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings_as_root.sh" && \ +"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_restore_spack_user_settings_as_root.sh" && \ "${SPACK_INSTALL_SCRIPTS}/generate_modules.sh" || \ ( sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/preserve_built_spack_packages.sh" && diff --git a/.ci/deploy_container.sh b/.ci/deploy_container.sh index 5c5e456fb211a8d9a95103e9a5211a9cd4aa1099..4b5d2e769f566533fc343efb40a1d4666d97053d 100755 --- a/.ci/deploy_container.sh +++ b/.ci/deploy_container.sh @@ -11,16 +11,22 @@ source "${SOURCE_DIR}/commons.sh" INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}" FALLBACK_DIR="${HOME}/container_mount_full" -IMAGE_NAME="singularity_spack_temp.img" +IMAGE_NAME="singularity_${CONTAINER_STYLE}_temp.img" DATE=$(date --iso) +declare -A CONTAINER_PREFIX_LUT +CONTAINER_PREFIX_LUT[visionary]="" +CONTAINER_PREFIX_LUT[asic]="asic" + +CONTAINER_PREFIX=${CONTAINER_PREFIX_LUT[$CONTAINER_STYLE]} + get_container_name() { local local_num="$1" if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then - echo -n "${INSTALL_DIR}/$(get_change_name)_${DATE}_${local_num}.img" + echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}$(get_change_name)_${DATE}_${local_num}.img" else - echo -n "${INSTALL_DIR}/${DATE}_${local_num}.img" + echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}${DATE}_${local_num}.img" fi } @@ -43,7 +49,7 @@ cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || ( ) if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then - ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/latest + ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}latest # Announce new container in "Building & Deployment" channel # Since the output of this script is used in other parts, we have to hide curl's output @@ -51,7 +57,7 @@ if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then export https_proxy=http://proxy.kip.uni-heidelberg.de:8080 curl -i -X POST -H 'Content-Type: application/json' \ - -d "{\"text\": \"@channel New stable container built: \`${CONTAINER_NAME}\`\"}" \ + -d "{\"text\": \"@channel New stable ${CONTAINER_PREFIX} container built: \`${CONTAINER_NAME}\`\"}" \ https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null # extract dna diff --git a/.ci/install_visionary_spack.sh b/.ci/install_spack.sh similarity index 100% rename from .ci/install_visionary_spack.sh rename to .ci/install_spack.sh diff --git a/.ci/create_visionary_recipe.sh b/.ci/visionary_create_recipe.sh similarity index 99% rename from .ci/create_visionary_recipe.sh rename to .ci/visionary_create_recipe.sh index 65b9665fc37ca92d29bf9718229db0aa3c33439f..02df86d9724f933aabf151b5e09ad35c1c58f99f 100755 --- a/.ci/create_visionary_recipe.sh +++ b/.ci/visionary_create_recipe.sh @@ -76,6 +76,7 @@ From: ${DOCKER_BASE_IMAGE} export DEPENDENCY_PYTHON3="${DEPENDENCY_PYTHON3}" export VISIONARY_GCC="${VISIONARY_GCC}" export VISIONARY_GCC_VERSION="${VISIONARY_GCC_VERSION}" + export CONTAINER_STYLE="${CONTAINER_STYLE}" # Improve efficiency by installing system packages in the background (even # though we set the number of worker to \$(nproc), often times - e.g. when # concretizing - only one process will be active.) diff --git a/.ci/prepare_spack_as_root.sh b/.ci/visionary_prepare_spack_as_root.sh similarity index 100% rename from .ci/prepare_spack_as_root.sh rename to .ci/visionary_prepare_spack_as_root.sh diff --git a/.ci/restore_spack_user_settings_as_root.sh b/.ci/visionary_restore_spack_user_settings_as_root.sh similarity index 100% rename from .ci/restore_spack_user_settings_as_root.sh rename to .ci/visionary_restore_spack_user_settings_as_root.sh