diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 69bfc25500e1d0fa8ef1e3a547894dcef7f7ab15..797907cd0673ef863e63b71e91f4f20079162873 100755
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -39,6 +39,7 @@ pipeline {
YASHCHIKI_SANDBOXES = "sandboxes"
YASHCHIKI_PROXY_HTTP = "http://proxy.kip.uni-heidelberg.de:8080"
YASHCHIKI_PROXY_HTTPS = "http://proxy.kip.uni-heidelberg.de:8080"
+ YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME = get_build_cache_on_failure_name()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
stages {
@@ -83,6 +84,16 @@ pipeline {
steps {
script {
try {
+ // extract options from gerrit comment
+ boolean with_debug = false
+ boolean with_spack_verbose = false
+ if (isTriggeredByGerrit()) {
+ gerrit_comment = jesh(script: "echo '${GERRIT_EVENT_COMMENT_TEXT}' | base64 -d", returnStdout: true)
+ with_debug = gerrit_comment.contains("WITH_DEBUG")
+ with_spack_verbose = gerrit_comment.contains("WITH_SPACK_VERBOSE")
+ env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
+ }
+
sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_visionary_temp.img " +
"--log-dir=log " +
"--proxy-http=${YASHCHIKI_PROXY_HTTP} " +
@@ -94,7 +105,10 @@ pipeline {
"--host-env-filename=${WORKSPACE}/host.env " +
"--build-cache-name=${BUILD_CACHE_NAME} " +
("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
- "--recipe-filename=${WORKSPACE}/visionary_recipe.def"
+ "--recipe-filename=${WORKSPACE}/visionary_recipe.def " +
+ "--build-cache-on-failure-name=${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME} " +
+ (with_debug ? "--debug " : "") +
+ (with_spack_verbose ? "--spack-verbose " : "")
} catch (Throwable t) {
archiveArtifacts "errors_concretization.log"
throw t
@@ -229,3 +243,7 @@ pipeline {
}
}
}
+
+String get_build_cache_on_failure_name() {
+ return (CONTAINER_BUILD_TYPE == "testing" ? "c${GERRIT_CHANGE_NUMBER}p${GERRIT_PATCHSET_NUMBER}" : jesh("echo 'stable_\$(date --iso)'", returnStdout: true))
+}
diff --git a/.ci/Jenkinsfile_asic b/.ci/Jenkinsfile_asic
index b7f8716e2ad6332098c318720472e13003fae119..045b42f8b510050dd10e7e95f8b1bde50b2badde 100755
--- a/.ci/Jenkinsfile_asic
+++ b/.ci/Jenkinsfile_asic
@@ -39,6 +39,7 @@ pipeline {
YASHCHIKI_SANDBOXES = "sandboxes"
YASHCHIKI_PROXY_HTTP = "http://proxy.kip.uni-heidelberg.de:8080"
YASHCHIKI_PROXY_HTTPS = "http://proxy.kip.uni-heidelberg.de:8080"
+ YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME = get_build_cache_on_failure_name()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
agent { label 'conviz1||conviz2' }
@@ -84,6 +85,16 @@ pipeline {
steps {
script {
try {
+ // extract options from gerrit comment
+ boolean with_debug = false
+ boolean with_spack_verbose = false
+ if (isTriggeredByGerrit()) {
+ gerrit_comment = jesh(script: "echo '${GERRIT_EVENT_COMMENT_TEXT}' | base64 -d", returnStdout: true)
+ with_debug = gerrit_comment.contains("WITH_DEBUG")
+ with_spack_verbose = gerrit_comment.contains("WITH_SPACK_VERBOSE")
+ env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
+ }
+
sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_asic_temp.img " +
"--log-dir=log " +
"--proxy-${YASHCHIKI_PROXY_HTTP} " +
@@ -96,6 +107,9 @@ pipeline {
"--host-env-filename=${WORKSPACE}/host.env " +
("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
"--recipe-filename=${WORKSPACE}/asic_recipe.def "
+ "--build-cache-on-failure-name=${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME} " +
+ (with_debug ? "--debug " : "") +
+ (with_spack_verbose ? "--spack-verbose " : "")
} catch (Throwable t) {
archiveArtifacts "errors_concretization.log"
throw t
@@ -157,3 +171,7 @@ pipeline {
}
}
}
+
+String get_build_cache_on_failure_name() {
+ return (CONTAINER_BUILD_TYPE == "testing" ? "c${GERRIT_CHANGE_NUMBER}p${GERRIT_PATCHSET_NUMBER}" : jesh("echo 'stable_\$(date --iso)'", returnStdout: true))
+}
diff --git a/bin/yashchiki b/bin/yashchiki
index e4c88342ad0fa79ba22d15eb0c6bd8c39070782d..f96f4c1c0ff6703a902f7dae9da216da8bfc17ef 100644
--- a/bin/yashchiki
+++ b/bin/yashchiki
@@ -97,7 +97,11 @@ parser.add_argument(
"--build-cache-name", type=str, default="default",
help="Name of build cache to use, resides under "
"<CACHES_DIR>/build_caches/<BUILD_CACHE_NAME>.")
-
+parser.add_argument(
+ "--build-cache-on-failure-name", type=str,
+ help="Name of build cache to create upon failure, resides under "
+ "<CACHES_DIR>/build_caches/failed/<BUILD_CACHE_NAME> and is "
+ "postfix'ed with an ascending integer.")
# optional options
parser.add_argument(
"--proxy-http", type=str,
@@ -108,6 +112,9 @@ parser.add_argument(
parser.add_argument(
"--debug", action="store_true",
help="Enable debug-level logging.")
+parser.add_argument(
+ "--spack-verbose", action="store_true",
+ help="Enable verbose spack logging.")
args = parser.parse_args()
@@ -157,6 +164,7 @@ env = {
"CONTAINER_STYLE": args.style,
"CONTAINER_BUILD_TYPE": "testing", # FIXME: should not be required
"YASHCHIKI_DEBUG": str(int(args.debug)),
+ "YASHCHIKI_SPACK_VERBOSE": str(int(args.spack_verbose)),
"YASHCHIKI_SANDBOXES": args.sandboxes_dir,
"YASHCHIKI_IMAGE_NAME": args.output,
"YASHCHIKI_SPACK_PATH": args.spack_dir,
@@ -173,6 +181,9 @@ if args.proxy_http:
if args.proxy_https:
env = env | {"YASHCHIKI_PROXY_HTTPS": args.proxy_https}
+if args.build_cache_on_failure_name:
+ env = env | {"YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME": args.build_cache_on_failure_name}
+
# create directory for logs
args.log_dir.mkdir(parents=True, exist_ok=True)
@@ -239,7 +250,7 @@ with tempfile.TemporaryDirectory(prefix="spack-", dir=env["TMPDIR"]) \
host_env_filename = temporary_directory.joinpath("host.env")
env = env | {"YASHCHIKI_HOST_ENV_PATH": host_env_filename}
- run("lib/yashchiki/validate_environment.sh", env)
+ run("lib/yashchiki/store_environment.sh", env)
run("lib/yashchiki/create_spack_user.sh", env)
run("lib/yashchiki/create_caches.sh", env)
run("lib/yashchiki/fetch.sh", env)
diff --git a/bin/yashchiki_create_temporary_build_cache_after_failure.sh b/bin/yashchiki_create_temporary_build_cache_after_failure.sh
index c9c28799ca03e2e6ccef5d257b8811fe5873690b..596b2ac91a2ef992386a60b9b079841c9551da2a 100755
--- a/bin/yashchiki_create_temporary_build_cache_after_failure.sh
+++ b/bin/yashchiki_create_temporary_build_cache_after_failure.sh
@@ -1,25 +1,16 @@
#!/bin/bash
# This script creates a temporary build cache via symlinks in
-# failed/c<num>p<num>_<num>.
+# failed/${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME}_<num>.
#
# It streamlines container deployment as every container deployment will
# restart at the last sucessfully built package.
#
# Echoes path to created cache relative to build_caches base directory.
-if [ "${CONTAINER_BUILD_TYPE}" = "stable" ]; then
- echo "Stable container creation failed, this should not happen." >&2
- exit 1
-fi
-
-ROOT_DIR="$(dirname "$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")")"
-source "${ROOT_DIR}/lib/yashchiki/get_change_name.sh"
-
build_num=1
-change_num="$(get_change_name)"
-while [ -d "${YASHCHIKI_CACHES_ROOT}/preserved_packages/${change_num}_${build_num}" ]; do
+while [ -d "${YASHCHIKI_CACHES_ROOT}/preserved_packages/${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME}_${build_num}" ]; do
(( build_num++ ))
done
# `build_num` is used to indicate if no preserved packages were found: After
@@ -29,8 +20,8 @@ done
# store in the failed cache.
(( build_num-- ))
-preserved_packages="${YASHCHIKI_CACHES_ROOT}/preserved_packages/${change_num}_${build_num}"
-failed_build_cache="${YASHCHIKI_CACHES_ROOT}/build_caches/failed/${change_num}_${build_num}"
+preserved_packages="${YASHCHIKI_CACHES_ROOT}/preserved_packages/${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME}_${build_num}"
+failed_build_cache="${YASHCHIKI_CACHES_ROOT}/build_caches/failed/${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME}_${build_num}"
# expects input to be \0-printed
link_into_failed_buildcache() {
diff --git a/bin/yashchiki_get_build_cache_name.sh b/bin/yashchiki_get_build_cache_name.sh
new file mode 100755
index 0000000000000000000000000000000000000000..fa2d6e6e9e7eca13cc60428186127cf362a25f21
--- /dev/null
+++ b/bin/yashchiki_get_build_cache_name.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+set -euo pipefail
+shopt -s inherit_errexit
+
+ROOT_DIR="$(dirname "$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")")"
+source "${ROOT_DIR}/lib/yashchiki/get_latest_failed_build_cache.sh"
+
+# For testing changesets, see if user supplied a custom build cache with
+# `WITH_CACHE_NAME=<name>`. If not, check if there is a saved build cache from a
+# previous build of this changeset and use that as build cache. If the comment
+# contains `WITHOUT_FAILED_CACHE` we do nothing, i.e. we use the default cache.
+if [ -n "${GERRIT_EVENT_COMMENT_TEXT:-}" ]; then
+
+ tmpfile_comment="$(mktemp)"
+
+ echo "${GERRIT_EVENT_COMMENT_TEXT}" | base64 -d > "${tmpfile_comment}"
+
+ if grep -q "\bWITH_CACHE_NAME=" "${tmpfile_comment}"; then
+ # use specified cache
+ echo "$(sed -nE \
+ -e "s:.*\<WITH_CACHE_NAME=(\S*)(\s|$).*:\1:gp" \
+ "${tmpfile_comment}")"
+ rm "${tmpfile_comment}"
+ exit 0
+ elif ! grep -q "\bWITHOUT_FAILED_CACHE\b" "${tmpfile_comment}"; then
+ latest_failed_build_cache="$(get_latest_failed_build_cache_name)"
+
+ # If there is no previous build cache the while loop will terminate
+ # immedately and build_num be zero.
+ if [ -n "${latest_failed_build_cache}" ]; then
+ echo "${latest_failed_build_cache}"
+ rm "${tmpfile_comment}"
+ exit 0
+ fi
+ fi
+
+ rm "${tmpfile_comment}"
+fi
+
+echo "${BUILD_CACHE_NAME}"
diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh
index b154a4a7ed946901d0c1fc38ea259b57f264d13a..b6c20e66a57f323da104ec3c429db20d25f13bc2 100755
--- a/lib/yashchiki/commons.sh
+++ b/lib/yashchiki/commons.sh
@@ -190,7 +190,7 @@ SPACK_ARGS_INSTALL=()
SPACK_ARGS_REINDEX=()
SPACK_ARGS_VIEW=()
-if [ -n "$(get_host_env SPACK_VERBOSE)" ]; then
+if [ -n "$(get_host_env YASHCHIKI_SPACK_VERBOSE)" ]; then
SPACK_ARGS_INSTALL+=("--verbose")
SPACK_ARGS_VIEW+=("--verbose")
SPACK_ARGS_REINDEX+=("--verbose")
@@ -512,64 +512,10 @@ _install_from_buildcache() {
${MY_SPACK_BIN} "${SPACK_ARGS_REINDEX[@]+"${SPACK_ARGS_REINDEX[@]}"}" reindex
}
-get_latest_failed_build_cache_name() {
- local full_change_num
- local possible_build_caches
- local latest_patch_level
- local latest_build_num
-
- full_change_num="$(get_change_name)"
- change_num="${full_change_num%%p*}"
- possible_build_caches="$(mktemp)"
-
- find "${BASE_BUILD_CACHE_FAILED_OUTSIDE}" -mindepth 1 -maxdepth 1 -type d -name "${change_num}*" -print0 \
- | xargs -r -n 1 -r -0 basename > "${possible_build_caches}"
-
- if (( $(wc -l <"${possible_build_caches}") == 0 )); then
- return 0
- fi
-
- latest_patch_level="$(cat "${possible_build_caches}" \
- | cut -d p -f 2 | cut -d _ -f 1 | sort -rg | head -n 1)"
-
- latest_build_num="$(grep "p${latest_patch_level}_" "${possible_build_caches}" \
- | cut -d _ -f 2 | sort -rg | head -n 1)"
-
- echo -n "failed/${change_num}p${latest_patch_level}_${latest_build_num}"
-
- rm "${possible_build_caches}"
-}
-
-
#############
# UTILITIES #
#############
-get_change_name() {
- local change_num
- local patch_level
-
- local gerrit_change_number
- local gerrit_patchset_number
- local gerrit_refspec
-
- gerrit_change_number="$(get_host_env GERRIT_CHANGE_NUMBER)"
- gerrit_patchset_number="$(get_host_env GERRIT_PATCHSET_NUMBER)"
- gerrit_refspec="$(get_host_env GERRIT_REFSPEC)"
-
- if [ -z "${gerrit_change_number:-}" ]; then
- if [ -n "${gerrit_refspec:-}" ]; then
- # extract gerrit change number from refspec
- change_num="$(echo "${gerrit_refspec}" | cut -f 4 -d / )"
- patch_level="$(echo "${gerrit_refspec}" | cut -f 5 -d / )"
- fi
- else
- change_num="${gerrit_change_number}"
- patch_level="${gerrit_patchset_number}"
- fi
- echo -n "c${change_num}p${patch_level}"
-}
-
# copied from slurmviz-commons.sh
get_latest_hash() {
# Usage: get_latest_hash <pkg-name>
diff --git a/lib/yashchiki/get_latest_failed_build_cache.sh b/lib/yashchiki/get_latest_failed_build_cache.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5850f9bc0d824c7c8ab80da5d84e999914d3b2a4
--- /dev/null
+++ b/lib/yashchiki/get_latest_failed_build_cache.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -euo pipefail
+shopt -s inherit_errexit 2>/dev/null || true
+
+ROOT_DIR="$(dirname "$(dirname "$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")")")"
+source ${ROOT_DIR}/lib/yashchiki/get_change_name.sh
+
+get_latest_failed_build_cache_name() {
+ local full_change_num
+ local possible_build_caches
+ local latest_patch_level
+ local latest_build_num
+
+ full_change_num="$(get_change_name)"
+ change_num="${full_change_num%%p*}"
+ possible_build_caches="$(mktemp)"
+
+ find "${YASHCHIKI_CACHES_ROOT}/build_caches/failed" -mindepth 1 -maxdepth 1 -type d -name "${change_num}*" -print0 \
+ | xargs -r -n 1 -r -0 basename > "${possible_build_caches}"
+
+ if (( $(wc -l <"${possible_build_caches}") == 0 )); then
+ rm "${possible_build_caches}"
+ return 0
+ fi
+
+ latest_patch_level="$(cat "${possible_build_caches}" \
+ | cut -d p -f 2 | cut -d _ -f 1 | sort -rg | head -n 1)"
+
+ latest_build_num="$(grep "p${latest_patch_level}_" "${possible_build_caches}" \
+ | cut -d _ -f 2 | sort -rg | head -n 1)"
+
+ echo -n "failed/${change_num}p${latest_patch_level}_${latest_build_num}"
+
+ rm "${possible_build_caches}"
+}
diff --git a/lib/yashchiki/preserve_built_spack_packages.sh b/lib/yashchiki/preserve_built_spack_packages.sh
index dba27fb3ba812a43301105ed21606ad3947fc9d1..5b87548527acdbacb9151f175def59b7b042c34e 100755
--- a/lib/yashchiki/preserve_built_spack_packages.sh
+++ b/lib/yashchiki/preserve_built_spack_packages.sh
@@ -23,16 +23,10 @@ sourcedir="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
source "${sourcedir}/commons.sh"
source "${sourcedir}/setup_env_spack.sh"
-build_type=$(get_host_env CONTAINER_BUILD_TYPE)
-
# find empty directory to dump into
build_num=1
while /bin/true; do
- if [ "${build_type}" = "testing" ]; then
- target_folder="${PRESERVED_PACKAGES_INSIDE}/$(get_change_name)_${build_num}"
- else
- target_folder="${PRESERVED_PACKAGES_INSIDE}/${build_type}_$(date --iso)_${build_num}"
- fi
+ target_folder="${PRESERVED_PACKAGES_INSIDE}/${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME}_${build_num}"
if [ ! -d "${target_folder}" ]; then
break
diff --git a/lib/yashchiki/store_environment.sh b/lib/yashchiki/store_environment.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9a459f10e8b9ae7977352446566273f8fb7987aa
--- /dev/null
+++ b/lib/yashchiki/store_environment.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+shopt -s inherit_errexit
+
+# store environment for usage within container
+echo "# Host environment set to:" >&2
+env | tee "${YASHCHIKI_HOST_ENV_PATH}"
diff --git a/lib/yashchiki/validate_environment.sh b/lib/yashchiki/validate_environment.sh
deleted file mode 100755
index f709216e36f247dacd0f6add09a022d57af8c3dc..0000000000000000000000000000000000000000
--- a/lib/yashchiki/validate_environment.sh
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-shopt -s inherit_errexit
-
-#
-# Some early checks to make sure all needed environment variables are defined.
-#
-if [ "${CONTAINER_BUILD_TYPE}" != "testing" ] && \
- [ "${CONTAINER_BUILD_TYPE}" != "stable" ]; then
- echo "CONTAINER_BUILD_TYPE needs to be 'testing' or 'stable'!" >&2
- exit 1
-fi
-
-if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
- # In case of testing builds we need to include change number and patchset
- # level into the final image name. Hence we check beforehand if we have all
- # information to generate the image name.
- #
- # We need to have either:
- # * both change number AND patchset number
- # * a refspec from which we extract changeset number and patchset
- # therefore we have to fail both cases fail.
- #
- if [[ ! (( -n "${GERRIT_CHANGE_NUMBER:-}"
- && -n "${GERRIT_PATCHSET_NUMBER:-}" )
- || -n "${GERRIT_REFSPEC:-}" ) ]]; then
- echo -n "Neither GERRIT_REFSPEC nor GERRIT_CHANGE_NUMBER/" >&2
- echo -n "GERRIT_PATCHSET_NUMBER specified " >&2
- echo "for testing build." >&2
- exit 1
- fi
-fi
-
-export SPACK_VERBOSE=
-# source file for early validation
-SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-source "${SOURCE_DIR}/commons.sh"
-
-# For testing changesets, see if user supplied a custom build cache with
-# `WITH_CACHE_NAME=<name>`. If not, check if there is a saved build cache from a
-# previous build of this changeset and use that as build cache. If the comment
-# contains `WITHOUT_FAILED_CACHE` we do nothing, i.e. we use the default cache.
-if [ "${CONTAINER_BUILD_TYPE}" = "testing" ] \
- && [ -n "${GERRIT_EVENT_COMMENT_TEXT:-}" ]; then
-
- tmpfile_comment="$(mktemp)"
-
- echo "${GERRIT_EVENT_COMMENT_TEXT}" | base64 -d > "${tmpfile_comment}"
-
- if grep -q "\bWITH_DEBUG\b" "${tmpfile_comment}"; then
- export YASHCHIKI_DEBUG=enabled
- set_debug_output_from_env
- else
- export YASHCHIKI_DEBUG=""
- fi
-
- if grep -q "\bWITH_CACHE_NAME=" "${tmpfile_comment}"; then
- # use specified cache
- BUILD_CACHE_NAME="$(sed -nE \
- -e "s:.*\<WITH_CACHE_NAME=(\S*)(\s|$).*:\1:gp" \
- "${tmpfile_comment}")"
- export BUILD_CACHE_NAME
- elif ! grep -q "\bWITHOUT_FAILED_CACHE\b" "${tmpfile_comment}"; then
- latest_failed_build_cache="$(get_latest_failed_build_cache_name)"
-
- # If there is no previous build cache the while loop will terminate
- # immedately and build_num be zero.
- if [ -n "${latest_failed_build_cache}" ]; then
- export BUILD_CACHE_NAME="${latest_failed_build_cache}"
- fi
- fi
-
- if grep -q "\bWITH_SPACK_VERBOSE\b" "${tmpfile_comment}"; then
- export SPACK_VERBOSE="enabled"
- else
- export SPACK_VERBOSE=""
- fi
-
- rm "${tmpfile_comment}"
-fi
-
-# store environment for usage within container
-echo "# Host environment set to:" >&2
-env | tee "${HOST_ENV_FILE}"