From f11729cb10a9394fc79ab37004bec2bce712a22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Fri, 30 Jul 2021 16:35:23 +0200
Subject: [PATCH] CI: Introduce `CONTAINER_STYLE` (asic/visionary)

Allows us to drop some redundant `asic`-specific files.

Change-Id: I83506cb451fc0dcd5de045cc7483af42881258a8
---
 .ci/Jenkinsfile                               |  8 ++-
 .ci/Jenkinsfile_asic                          | 11 +++-
 .ci/asic_build_image.sh                       | 23 -------
 .ci/asic_build_sandbox.sh                     | 36 -----------
 .ci/asic_deploy_container.sh                  | 61 -------------------
 .ci/bootstrap_spack.sh                        | 42 +++++++------
 .ci/build_image.sh                            |  6 +-
 .ci/build_sandbox.sh                          | 12 ++--
 ..._install_routine_called_in_post_as_root.sh |  6 +-
 .ci/deploy_container.sh                       | 16 +++--
 ...ll_visionary_spack.sh => install_spack.sh} |  0
 ...y_recipe.sh => visionary_create_recipe.sh} |  1 +
 ....sh => visionary_prepare_spack_as_root.sh} |  0
 ...ry_restore_spack_user_settings_as_root.sh} |  0
 14 files changed, 60 insertions(+), 162 deletions(-)
 delete mode 100755 .ci/asic_build_image.sh
 delete mode 100755 .ci/asic_build_sandbox.sh
 delete mode 100755 .ci/asic_deploy_container.sh
 rename .ci/{install_visionary_spack.sh => install_spack.sh} (100%)
 rename .ci/{create_visionary_recipe.sh => visionary_create_recipe.sh} (99%)
 rename .ci/{prepare_spack_as_root.sh => visionary_prepare_spack_as_root.sh} (100%)
 rename .ci/{restore_spack_user_settings_as_root.sh => visionary_restore_spack_user_settings_as_root.sh} (100%)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 219a1e5a..4891f2d0 100755
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -27,6 +27,10 @@ pipeline {
 			   description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
 	}
 
+	environment {
+		CONTAINER_STYLE = "visionary"
+	}
+
 	stages {
 		stage('Container Build') {
 			agent { label 'conviz1||conviz2' }
@@ -43,7 +47,7 @@ pipeline {
 				VISIONARY_GCC = "gcc@${VISIONARY_GCC_VERSION}"
 				TMPDIR = "/tmp/${env.NODE_NAME}"
 				JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
-								   returnStdout: true).trim()
+				                   returnStdout: true).trim()
 				BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}"  // propagate parameter to environment
 			}
 			stages {
@@ -98,7 +102,7 @@ pipeline {
 				}
 				stage('Create visionary recipe') {
 					steps {
-						sh ".ci/create_visionary_recipe.sh"
+						sh ".ci/visionary_create_recipe.sh"
 					}
 				}
 				stage('Build sandbox') {
diff --git a/.ci/Jenkinsfile_asic b/.ci/Jenkinsfile_asic
index 59f5ec32..b4d58ec9 100755
--- a/.ci/Jenkinsfile_asic
+++ b/.ci/Jenkinsfile_asic
@@ -16,6 +16,10 @@ pipeline {
 		skipDefaultCheckout()
 	}
 
+	environment {
+		CONTAINER_STYLE = "asic"
+	}
+
 	stages {
 		stage('Container Build') {
 			agent { label 'conviz1||conviz2' }
@@ -24,6 +28,7 @@ pipeline {
 				TMPDIR = "/tmp/${env.NODE_NAME}"
 				JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
 				                   returnStdout: true).trim()
+				BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}"  // propagate parameter to environment
 			}
 			stages {
 				stage('Pre-build Cleanup') {
@@ -46,19 +51,19 @@ pipeline {
 				}
 				stage('Build sandbox') {
 					steps {
-						sh ".ci/asic_build_sandbox.sh"
+						sh ".ci/build_sandbox.sh"
 					}
 				}
 				stage('Build container image') {
 					steps {
-						sh ".ci/asic_build_image.sh"
+						sh ".ci/build_image.sh"
 					}
 				}
 				stage('Update build cache and export container') {
 					steps {
 						script {
 							// we only want the container name, tail everything else
-							CONTAINER_NAME = sh(script: ".ci/asic_deploy_container.sh | tail -n 1", returnStdout: true).trim()
+							CONTAINER_IMAGE = sh(script: ".ci/deploy_container.sh | tail -n 1", returnStdout: true).trim()
 						}
 					}
 				}
diff --git a/.ci/asic_build_image.sh b/.ci/asic_build_image.sh
deleted file mode 100755
index d0144a2c..00000000
--- a/.ci/asic_build_image.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-shopt -s inherit_errexit
-
-if (( $(find sandboxes -mindepth 1 -maxdepth 1 | wc -l) > 1 )); then
-    echo "More than one sandbox found, this should never happen!" >&2
-    exit 1
-fi
-
-TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)"
-
-# create image file
-IMAGE_NAME=singularity_asic_temp.img
-
-sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}"
-
-sudo chown -R vis_jenkins singularity_asic_*.img
-
-if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then
-    # allow spack user to execute image
-    chmod +rx singularity_asic_*.img
-fi
diff --git a/.ci/asic_build_sandbox.sh b/.ci/asic_build_sandbox.sh
deleted file mode 100755
index 2c427712..00000000
--- a/.ci/asic_build_sandbox.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-shopt -s inherit_errexit
-
-# set generic locale for building
-export LANG=C.UTF-8
-export LC_ALL=C.UTF-8
-
-unset LC_CTYPE
-unset LC_COLLATE
-unset LC_MONETARY
-unset LC_NUMERIC
-unset LC_TIME
-unset LC_MESSAGES
-
-echo "creating asic_recipe.def" >&2
-SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-"${SOURCE_DIR}/asic_create_recipe.sh"
-
-echo "created asic_recipe.def" >&2
-cat "${WORKSPACE}/asic_recipe.def"
-
-# build the container (using scripts from above)
-export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
-export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
-
-TARGET_FOLDER="sandboxes/asic"
-
-# Do not change: special sudo permit for jenkins user...
-sudo rm -rf sandboxes/
-
-mkdir sandboxes
-
-# Do not change: special sudo permit for jenkins user...
-sudo -E singularity build --sandbox "${TARGET_FOLDER}" asic_recipe.def
diff --git a/.ci/asic_deploy_container.sh b/.ci/asic_deploy_container.sh
deleted file mode 100755
index 735599f8..00000000
--- a/.ci/asic_deploy_container.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-shopt -s inherit_errexit
-
-# NOTE: stdout of this script is parsed within the Jenkinsfile.
-#       Think twice before adding any output!
-
-SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-source "${SOURCE_DIR}/asic_dummy_variables.sh"
-source "${SOURCE_DIR}/commons.sh"
-
-INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}"
-FALLBACK_DIR="${HOME}/container_mount_full"
-IMAGE_NAME="singularity_asic_temp.img"
-DATE=$(date --iso)
-
-get_container_name()
-{
-    local local_num="$1"
-    if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
-        echo -n "${INSTALL_DIR}/asic_$(get_change_name)_${DATE}_${local_num}.img"
-    else
-        echo -n "${INSTALL_DIR}/asic_${DATE}_${local_num}.img"
-    fi
-}
-
-# find unused image name
-num=1
-while [[ -e "$(get_container_name ${num})" ]] ; do
-    (( num++ ))
-done
-
-CONTAINER_NAME="$(get_container_name ${num})"
-# this must be the only output for Jenkins to pick up the container path
-# in order to trigger the downstream software builds in the correct container
-echo $CONTAINER_NAME
-
-# copy to target
-cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || (
-    echo "Error: Copy failed because the mount point is full, saving container image to fallback location.." >&2
-    cp -v "${IMAGE_NAME}" "${FALLBACK_DIR}/$(basename "${CONTAINER_NAME}")" >&2
-    exit 1
-)
-
-if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
-    ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/asic_latest
-
-    # Announce new container in "Building & Deployment" channel
-    # Since the output of this script is used in other parts, we have to hide curl's output
-    export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
-    export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
-
-    curl -i -X POST -H 'Content-Type: application/json' \
-        -d "{\"text\": \"@channel New stable ASIC container built: \`${CONTAINER_NAME}\`\"}" \
-        https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null
-fi
-
-# delete temporary image
-rm "${IMAGE_NAME}"
-
diff --git a/.ci/bootstrap_spack.sh b/.ci/bootstrap_spack.sh
index d349008d..4774f015 100755
--- a/.ci/bootstrap_spack.sh
+++ b/.ci/bootstrap_spack.sh
@@ -58,23 +58,25 @@ echo ""
     exit 1
 fi
 
-# check if it can be specialized
-spec_compiler="${VISIONARY_GCC}"
-install_from_buildcache "${spec_compiler}"
-
-# remember system compiler versions (to be removed later)
-system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)"
-
-# upgrade to newer gcc
-echo "INSTALL NEW GCC"
-set -x
-${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}"
-
-# remove system compilers from spack to avoid conflicting concretization
-echo "$(${MY_SPACK_BIN} compiler list)"
-for system_compiler in ${system_compilers}; do
-    ${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}"
-done
-
-# add fresh compiler to spack
-${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-*
+if [ "${CONTAINER_STYLE}" = "visionary" ]; then
+    # check if it can be specialized
+    spec_compiler="${VISIONARY_GCC}"
+    install_from_buildcache "${spec_compiler}"
+
+    # remember system compiler versions (to be removed later)
+    system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)"
+
+    # upgrade to newer gcc
+    echo "INSTALL NEW GCC"
+    set -x
+    ${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}"
+
+    # remove system compilers from spack to avoid conflicting concretization
+    echo "$(${MY_SPACK_BIN} compiler list)"
+    for system_compiler in ${system_compilers}; do
+        ${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}"
+    done
+
+    # add fresh compiler to spack
+    ${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-*
+fi
diff --git a/.ci/build_image.sh b/.ci/build_image.sh
index 020161a6..6fcac061 100755
--- a/.ci/build_image.sh
+++ b/.ci/build_image.sh
@@ -11,7 +11,7 @@ fi
 TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)"
 
 # create image file
-IMAGE_NAME=singularity_spack_temp.img
+IMAGE_NAME=singularity_${CONTAINER_STYLE}_temp.img
 
 # We want the spack folder to be available inside the container image
 # -> it needs to be bind mounted to the sandbox folder
@@ -22,9 +22,9 @@ sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}"
 # umount spack folder afterwards
 sudo umount "${TARGET_FOLDER}/opt/spack"
 
-sudo chown -R vis_jenkins singularity_spack_*.img
+sudo chown -R vis_jenkins singularity_${CONTAINER_STYLE}_*.img
 
 if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then
     # allow spack user to execute image
-    chmod +rx singularity_spack_*.img
+    chmod +rx singularity_${CONTAINER_STYLE}_*.img
 fi
diff --git a/.ci/build_sandbox.sh b/.ci/build_sandbox.sh
index e32af6d6..299bf95a 100755
--- a/.ci/build_sandbox.sh
+++ b/.ci/build_sandbox.sh
@@ -14,12 +14,12 @@ unset LC_NUMERIC
 unset LC_TIME
 unset LC_MESSAGES
 
-echo "creating visionary_recipe.def" >&2
+echo "creating ${CONTAINER_STYLE}_recipe.def" >&2
 SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-"${SOURCE_DIR}/create_visionary_recipe.sh"
+"${SOURCE_DIR}/${CONTAINER_STYLE}_create_recipe.sh"
 
-echo "created visionary_recipe.def" >&2
-cat "${WORKSPACE}/visionary_recipe.def"
+echo "created ${CONTAINER_STYLE}_recipe.def" >&2
+cat "${WORKSPACE}/${CONTAINER_STYLE}_recipe.def"
 
 # create some jenkins-owned temp folder for spack build
 if [ ! -d "${JOB_TMP_SPACK}" ]; then
@@ -34,7 +34,7 @@ chmod 777 "${JOB_TMP_SPACK}"
 export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
 export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
 
-TARGET_FOLDER="sandboxes/buster_spack"
+TARGET_FOLDER="sandboxes/${CONTAINER_STYLE}"
 
 # Do not change: special sudo permit for jenkins user...
 sudo rm -rf sandboxes/
@@ -42,4 +42,4 @@ sudo rm -rf sandboxes/
 mkdir sandboxes
 
 # Do not change: special sudo permit for jenkins user...
-sudo -E singularity build --sandbox "${TARGET_FOLDER}" visionary_recipe.def | tee out_singularity_build_visionary_recipe.txt
+sudo -E singularity build --sandbox "${TARGET_FOLDER}" ${CONTAINER_STYLE}_recipe.def | tee out_singularity_build_${CONTAINER_STYLE}_recipe.txt
diff --git a/.ci/complete_spack_install_routine_called_in_post_as_root.sh b/.ci/complete_spack_install_routine_called_in_post_as_root.sh
index 9aca0cb6..9f9426b7 100755
--- a/.ci/complete_spack_install_routine_called_in_post_as_root.sh
+++ b/.ci/complete_spack_install_routine_called_in_post_as_root.sh
@@ -11,16 +11,16 @@ shopt -s inherit_errexit 2>/dev/null || true
 SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
 source "${SOURCE_DIR}/commons.sh"
 
-"${SPACK_INSTALL_SCRIPTS}/prepare_spack_as_root.sh"
+"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_prepare_spack_as_root.sh"
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/bootstrap_spack.sh"
 
-sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_visionary_spack.sh" && \
+sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_spack.sh" && \
 sudo -Eu spack ${MY_SPACK_BIN} compiler add --scope site /usr/bin && \
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings.sh" && \
 # remove temporary cache folder
 rm -rfv /opt/spack/.spack
 
-"${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings_as_root.sh" && \
+"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_restore_spack_user_settings_as_root.sh" && \
 "${SPACK_INSTALL_SCRIPTS}/generate_modules.sh" || \
 (
 sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/preserve_built_spack_packages.sh" &&
diff --git a/.ci/deploy_container.sh b/.ci/deploy_container.sh
index 5c5e456f..4b5d2e76 100755
--- a/.ci/deploy_container.sh
+++ b/.ci/deploy_container.sh
@@ -11,16 +11,22 @@ source "${SOURCE_DIR}/commons.sh"
 
 INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}"
 FALLBACK_DIR="${HOME}/container_mount_full"
-IMAGE_NAME="singularity_spack_temp.img"
+IMAGE_NAME="singularity_${CONTAINER_STYLE}_temp.img"
 DATE=$(date --iso)
 
+declare -A CONTAINER_PREFIX_LUT
+CONTAINER_PREFIX_LUT[visionary]=""
+CONTAINER_PREFIX_LUT[asic]="asic"
+
+CONTAINER_PREFIX=${CONTAINER_PREFIX_LUT[$CONTAINER_STYLE]}
+
 get_container_name()
 {
     local local_num="$1"
     if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
-        echo -n "${INSTALL_DIR}/$(get_change_name)_${DATE}_${local_num}.img"
+        echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}$(get_change_name)_${DATE}_${local_num}.img"
     else
-        echo -n "${INSTALL_DIR}/${DATE}_${local_num}.img"
+        echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}${DATE}_${local_num}.img"
     fi
 }
 
@@ -43,7 +49,7 @@ cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || (
 )
 
 if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
-    ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/latest
+    ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}latest
 
     # Announce new container in "Building & Deployment" channel
     # Since the output of this script is used in other parts, we have to hide curl's output
@@ -51,7 +57,7 @@ if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
     export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
 
     curl -i -X POST -H 'Content-Type: application/json' \
-        -d "{\"text\": \"@channel New stable container built: \`${CONTAINER_NAME}\`\"}" \
+        -d "{\"text\": \"@channel New stable ${CONTAINER_PREFIX} container built: \`${CONTAINER_NAME}\`\"}" \
         https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null
 
     # extract dna
diff --git a/.ci/install_visionary_spack.sh b/.ci/install_spack.sh
similarity index 100%
rename from .ci/install_visionary_spack.sh
rename to .ci/install_spack.sh
diff --git a/.ci/create_visionary_recipe.sh b/.ci/visionary_create_recipe.sh
similarity index 99%
rename from .ci/create_visionary_recipe.sh
rename to .ci/visionary_create_recipe.sh
index 65b9665f..02df86d9 100755
--- a/.ci/create_visionary_recipe.sh
+++ b/.ci/visionary_create_recipe.sh
@@ -76,6 +76,7 @@ From: ${DOCKER_BASE_IMAGE}
     export DEPENDENCY_PYTHON3="${DEPENDENCY_PYTHON3}"
     export VISIONARY_GCC="${VISIONARY_GCC}"
     export VISIONARY_GCC_VERSION="${VISIONARY_GCC_VERSION}"
+    export CONTAINER_STYLE="${CONTAINER_STYLE}"
     # Improve efficiency by installing system packages in the background (even
     # though we set the number of worker to \$(nproc), often times - e.g. when
     # concretizing - only one process will be active.)
diff --git a/.ci/prepare_spack_as_root.sh b/.ci/visionary_prepare_spack_as_root.sh
similarity index 100%
rename from .ci/prepare_spack_as_root.sh
rename to .ci/visionary_prepare_spack_as_root.sh
diff --git a/.ci/restore_spack_user_settings_as_root.sh b/.ci/visionary_restore_spack_user_settings_as_root.sh
similarity index 100%
rename from .ci/restore_spack_user_settings_as_root.sh
rename to .ci/visionary_restore_spack_user_settings_as_root.sh
-- 
GitLab