Skip to content
Snippets Groups Projects
Commit f11729cb authored by Eric Müller's avatar Eric Müller :mountain_bicyclist:
Browse files

CI: Introduce `CONTAINER_STYLE` (asic/visionary) 

Allows us to drop some redundant `asic`-specific files.

Change-Id: I83506cb451fc0dcd5de045cc7483af42881258a8
parent 4d111e5a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 60 additions and 162 deletions
.ci/Jenkinsfile
+ 6
− 2
View file @ f11729cb
......@@ -27,6 +27,10 @@ pipeline {
description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
}
environment {
CONTAINER_STYLE = "visionary"
}
stages {
stage('Container Build') {
agent { label 'conviz1||conviz2' }
......@@ -43,7 +47,7 @@ pipeline {
VISIONARY_GCC = "gcc@${VISIONARY_GCC_VERSION}"
TMPDIR = "/tmp/${env.NODE_NAME}"
JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
returnStdout: true).trim()
returnStdout: true).trim()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
stages {
......@@ -98,7 +102,7 @@ pipeline {
}
stage('Create visionary recipe') {
steps {
sh ".ci/create_visionary_recipe.sh"
sh ".ci/visionary_create_recipe.sh"
}
}
stage('Build sandbox') {
......
.ci/Jenkinsfile_asic
+ 8
− 3
View file @ f11729cb
......@@ -16,6 +16,10 @@ pipeline {
skipDefaultCheckout()
}
environment {
CONTAINER_STYLE = "asic"
}
stages {
stage('Container Build') {
agent { label 'conviz1||conviz2' }
......@@ -24,6 +28,7 @@ pipeline {
TMPDIR = "/tmp/${env.NODE_NAME}"
JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
returnStdout: true).trim()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
stages {
stage('Pre-build Cleanup') {
......@@ -46,19 +51,19 @@ pipeline {
}
stage('Build sandbox') {
steps {
sh ".ci/asic_build_sandbox.sh"
sh ".ci/build_sandbox.sh"
}
}
stage('Build container image') {
steps {
sh ".ci/asic_build_image.sh"
sh ".ci/build_image.sh"
}
}
stage('Update build cache and export container') {
steps {
script {
// we only want the container name, tail everything else
CONTAINER_NAME = sh(script: ".ci/asic_deploy_container.sh | tail -n 1", returnStdout: true).trim()
CONTAINER_IMAGE = sh(script: ".ci/deploy_container.sh | tail -n 1", returnStdout: true).trim()
}
}
}
......
.ci/asic_build_image.sh deleted 100755 → 0
+ 0
− 23
View file @ 4d111e5a
#!/bin/bash
set -euo pipefail
shopt -s inherit_errexit
if (( $(find sandboxes -mindepth 1 -maxdepth 1 | wc -l) > 1 )); then
echo "More than one sandbox found, this should never happen!" >&2
exit 1
fi
TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)"
# create image file
IMAGE_NAME=singularity_asic_temp.img
sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}"
sudo chown -R vis_jenkins singularity_asic_*.img
if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then
# allow spack user to execute image
chmod +rx singularity_asic_*.img
fi
.ci/asic_build_sandbox.sh deleted 100755 → 0
+ 0
− 36
View file @ 4d111e5a
#!/bin/bash
set -euo pipefail
shopt -s inherit_errexit
# set generic locale for building
export LANG=C.UTF-8
export LC_ALL=C.UTF-8
unset LC_CTYPE
unset LC_COLLATE
unset LC_MONETARY
unset LC_NUMERIC
unset LC_TIME
unset LC_MESSAGES
echo "creating asic_recipe.def" >&2
SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
"${SOURCE_DIR}/asic_create_recipe.sh"
echo "created asic_recipe.def" >&2
cat "${WORKSPACE}/asic_recipe.def"
# build the container (using scripts from above)
export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
TARGET_FOLDER="sandboxes/asic"
# Do not change: special sudo permit for jenkins user...
sudo rm -rf sandboxes/
mkdir sandboxes
# Do not change: special sudo permit for jenkins user...
sudo -E singularity build --sandbox "${TARGET_FOLDER}" asic_recipe.def
.ci/asic_deploy_container.sh deleted 100755 → 0
+ 0
− 61
View file @ 4d111e5a
#!/bin/bash
set -euo pipefail
shopt -s inherit_errexit
# NOTE: stdout of this script is parsed within the Jenkinsfile.
# Think twice before adding any output!
SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
source "${SOURCE_DIR}/asic_dummy_variables.sh"
source "${SOURCE_DIR}/commons.sh"
INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}"
FALLBACK_DIR="${HOME}/container_mount_full"
IMAGE_NAME="singularity_asic_temp.img"
DATE=$(date --iso)
get_container_name()
{
local local_num="$1"
if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
echo -n "${INSTALL_DIR}/asic_$(get_change_name)_${DATE}_${local_num}.img"
else
echo -n "${INSTALL_DIR}/asic_${DATE}_${local_num}.img"
fi
}
# find unused image name
num=1
while [[ -e "$(get_container_name ${num})" ]] ; do
(( num++ ))
done
CONTAINER_NAME="$(get_container_name ${num})"
# this must be the only output for Jenkins to pick up the container path
# in order to trigger the downstream software builds in the correct container
echo $CONTAINER_NAME
# copy to target
cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || (
echo "Error: Copy failed because the mount point is full, saving container image to fallback location.." >&2
cp -v "${IMAGE_NAME}" "${FALLBACK_DIR}/$(basename "${CONTAINER_NAME}")" >&2
exit 1
)
if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/asic_latest
# Announce new container in "Building & Deployment" channel
# Since the output of this script is used in other parts, we have to hide curl's output
export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
curl -i -X POST -H 'Content-Type: application/json' \
-d "{\"text\": \"@channel New stable ASIC container built: \`${CONTAINER_NAME}\`\"}" \
https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null
fi
# delete temporary image
rm "${IMAGE_NAME}"
.ci/bootstrap_spack.sh
+ 22
− 20
View file @ f11729cb
......@@ -58,23 +58,25 @@ echo ""
exit 1
fi
# check if it can be specialized
spec_compiler="${VISIONARY_GCC}"
install_from_buildcache "${spec_compiler}"
# remember system compiler versions (to be removed later)
system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)"
# upgrade to newer gcc
echo "INSTALL NEW GCC"
set -x
${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}"
# remove system compilers from spack to avoid conflicting concretization
echo "$(${MY_SPACK_BIN} compiler list)"
for system_compiler in ${system_compilers}; do
${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}"
done
# add fresh compiler to spack
${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-*
if [ "${CONTAINER_STYLE}" = "visionary" ]; then
# check if it can be specialized
spec_compiler="${VISIONARY_GCC}"
install_from_buildcache "${spec_compiler}"
# remember system compiler versions (to be removed later)
system_compilers="$(${MY_SPACK_BIN} compiler list --scope site | grep \@)"
# upgrade to newer gcc
echo "INSTALL NEW GCC"
set -x
${MY_SPACK_BIN} "${SPACK_ARGS_INSTALL[@]}" install --no-cache --show-log-on-error "${spec_compiler}"
# remove system compilers from spack to avoid conflicting concretization
echo "$(${MY_SPACK_BIN} compiler list)"
for system_compiler in ${system_compilers}; do
${MY_SPACK_BIN} compiler rm --scope site "${system_compiler}"
done
# add fresh compiler to spack
${MY_SPACK_BIN} compiler add --scope site ${MY_SPACK_FOLDER}/opt/spack/linux-*/*/gcc-${VISIONARY_GCC_VERSION}-*
fi
.ci/build_image.sh
+ 3
− 3
View file @ f11729cb
......@@ -11,7 +11,7 @@ fi
TARGET_FOLDER="$(find sandboxes -mindepth 1 -maxdepth 1)"
# create image file
IMAGE_NAME=singularity_spack_temp.img
IMAGE_NAME=singularity_${CONTAINER_STYLE}_temp.img
# We want the spack folder to be available inside the container image
# -> it needs to be bind mounted to the sandbox folder
......@@ -22,9 +22,9 @@ sudo singularity build ${IMAGE_NAME} "${TARGET_FOLDER}"
# umount spack folder afterwards
sudo umount "${TARGET_FOLDER}/opt/spack"
sudo chown -R vis_jenkins singularity_spack_*.img
sudo chown -R vis_jenkins singularity_${CONTAINER_STYLE}_*.img
if [[ "${CONTAINER_BUILD_TYPE}" =~ "^stable$" ]]; then
# allow spack user to execute image
chmod +rx singularity_spack_*.img
chmod +rx singularity_${CONTAINER_STYLE}_*.img
fi
.ci/build_sandbox.sh
+ 6
− 6
View file @ f11729cb
......@@ -14,12 +14,12 @@ unset LC_NUMERIC
unset LC_TIME
unset LC_MESSAGES
echo "creating visionary_recipe.def" >&2
echo "creating ${CONTAINER_STYLE}_recipe.def" >&2
SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
"${SOURCE_DIR}/create_visionary_recipe.sh"
"${SOURCE_DIR}/${CONTAINER_STYLE}_create_recipe.sh"
echo "created visionary_recipe.def" >&2
cat "${WORKSPACE}/visionary_recipe.def"
echo "created ${CONTAINER_STYLE}_recipe.def" >&2
cat "${WORKSPACE}/${CONTAINER_STYLE}_recipe.def"
# create some jenkins-owned temp folder for spack build
if [ ! -d "${JOB_TMP_SPACK}" ]; then
......@@ -34,7 +34,7 @@ chmod 777 "${JOB_TMP_SPACK}"
export http_proxy=http://proxy.kip.uni-heidelberg.de:8080
export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
TARGET_FOLDER="sandboxes/buster_spack"
TARGET_FOLDER="sandboxes/${CONTAINER_STYLE}"
# Do not change: special sudo permit for jenkins user...
sudo rm -rf sandboxes/
......@@ -42,4 +42,4 @@ sudo rm -rf sandboxes/
mkdir sandboxes
# Do not change: special sudo permit for jenkins user...
sudo -E singularity build --sandbox "${TARGET_FOLDER}" visionary_recipe.def | tee out_singularity_build_visionary_recipe.txt
sudo -E singularity build --sandbox "${TARGET_FOLDER}" ${CONTAINER_STYLE}_recipe.def | tee out_singularity_build_${CONTAINER_STYLE}_recipe.txt
.ci/complete_spack_install_routine_called_in_post_as_root.sh
+ 3
− 3
View file @ f11729cb
......@@ -11,16 +11,16 @@ shopt -s inherit_errexit 2>/dev/null || true
SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
source "${SOURCE_DIR}/commons.sh"
"${SPACK_INSTALL_SCRIPTS}/prepare_spack_as_root.sh"
"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_prepare_spack_as_root.sh"
sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/bootstrap_spack.sh"
sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_visionary_spack.sh" && \
sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/install_spack.sh" && \
sudo -Eu spack ${MY_SPACK_BIN} compiler add --scope site /usr/bin && \
sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings.sh" && \
# remove temporary cache folder
rm -rfv /opt/spack/.spack
"${SPACK_INSTALL_SCRIPTS}/restore_spack_user_settings_as_root.sh" && \
"${SPACK_INSTALL_SCRIPTS}/${CONTAINER_PREFIX}_restore_spack_user_settings_as_root.sh" && \
"${SPACK_INSTALL_SCRIPTS}/generate_modules.sh" || \
(
sudo -Eu spack "${SPACK_INSTALL_SCRIPTS}/preserve_built_spack_packages.sh" &&
......
.ci/deploy_container.sh
+ 11
− 5
View file @ f11729cb
......@@ -11,16 +11,22 @@ source "${SOURCE_DIR}/commons.sh"
INSTALL_DIR="/containers/${CONTAINER_BUILD_TYPE}"
FALLBACK_DIR="${HOME}/container_mount_full"
IMAGE_NAME="singularity_spack_temp.img"
IMAGE_NAME="singularity_${CONTAINER_STYLE}_temp.img"
DATE=$(date --iso)
declare -A CONTAINER_PREFIX_LUT
CONTAINER_PREFIX_LUT[visionary]=""
CONTAINER_PREFIX_LUT[asic]="asic"
CONTAINER_PREFIX=${CONTAINER_PREFIX_LUT[$CONTAINER_STYLE]}
get_container_name()
{
local local_num="$1"
if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
echo -n "${INSTALL_DIR}/$(get_change_name)_${DATE}_${local_num}.img"
echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}$(get_change_name)_${DATE}_${local_num}.img"
else
echo -n "${INSTALL_DIR}/${DATE}_${local_num}.img"
echo -n "${INSTALL_DIR}/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}${DATE}_${local_num}.img"
fi
}
......@@ -43,7 +49,7 @@ cp "${IMAGE_NAME}" "${CONTAINER_NAME}" || (
)
if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/latest
ln -sf "./$(basename ${CONTAINER_NAME})" /containers/stable/${CONTAINER_PREFIX}${CONTAINER_PREFIX:+_}latest
# Announce new container in "Building & Deployment" channel
# Since the output of this script is used in other parts, we have to hide curl's output
......@@ -51,7 +57,7 @@ if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
export https_proxy=http://proxy.kip.uni-heidelberg.de:8080
curl -i -X POST -H 'Content-Type: application/json' \
-d "{\"text\": \"@channel New stable container built: \`${CONTAINER_NAME}\`\"}" \
-d "{\"text\": \"@channel New stable ${CONTAINER_PREFIX} container built: \`${CONTAINER_NAME}\`\"}" \
https://chat.bioai.eu/hooks/iuhwp9k3h38c3d98uhwh5fxe9h &>/dev/null
# extract dna
......
.ci/install_visionary_spack.sh → .ci/install_spack.sh
+ 0
− 0
View file @ f11729cb
File moved
.ci/create_visionary_recipe.sh → .ci/visionary_create_recipe.sh
+ 1
− 0
View file @ f11729cb
......@@ -76,6 +76,7 @@ From: ${DOCKER_BASE_IMAGE}
export DEPENDENCY_PYTHON3="${DEPENDENCY_PYTHON3}"
export VISIONARY_GCC="${VISIONARY_GCC}"
export VISIONARY_GCC_VERSION="${VISIONARY_GCC_VERSION}"
export CONTAINER_STYLE="${CONTAINER_STYLE}"
# Improve efficiency by installing system packages in the background (even
# though we set the number of worker to \$(nproc), often times - e.g. when
# concretizing - only one process will be active.)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment