diff --git a/bin/yashchiki b/bin/yashchiki
index 90cd7a5ba6425920443a3c52ab199045d97bab33..b56d83f21c66846d1e06ef85f8a08829790fc3d7 100755
--- a/bin/yashchiki
+++ b/bin/yashchiki
@@ -167,6 +167,7 @@ env = os.environ.copy()
env.update({
"DOCKER_BASE_IMAGE": config["docker_base_image"],
"BUILD_BASE_SANDBOX": str(int(config["build_base_sandbox"])),
+ "SPACK_ENVIRONMENT": str(int(config["spack_environment"])),
# This needs to be here because otherwise the default python
# (2.7.18) will pollute the spec and lead to a conflict
# can be removed as soon as the explicit preferred version
diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh
index a770878dca8d089a4459939e886e1e885a8a02a1..aa2c6c46a6864156e79b88dc822a31ce79626fc7 100755
--- a/lib/yashchiki/commons.sh
+++ b/lib/yashchiki/commons.sh
@@ -237,6 +237,15 @@ get_specfile_name() {
awk '{ print "spec_" $1 ".yaml" }')"
}
+# Get the name of given environment, should only be called if one does not
+# depend on the specfile existing or having correct content, see
+# get_specfiles().
+get_specfile_name_for_environment() {
+ echo -n "${SPEC_FOLDER}/$(echo "environment_$1" | sha256sum |
+ awk '{ print "spec_" $1 ".yaml" }')"
+}
+
+
# Compute the concrete specfile for the given packages.
#
# Spec files are only computed once and afterwards their names are emitted via
@@ -291,13 +300,38 @@ get_specfiles() {
done
}
+# cf. get_specfiles but for spack environments
+get_specfiles_for_environments() {
+ local specfiles=()
+
+ for environment in "${@}"; do
+ # compute spec and put into temporary file derived from package name
+ specfiles+=("$(get_specfile_name_for_environment "${environment}")")
+ done
+
+ (
+ local idx=0
+ for package in "${@}"; do
+ if [ ! -f "${specfiles[${idx}]}" ]; then
+ echo "${MY_SPACK_CMD} -e \"${environment}\" spec -y > ${specfiles[${idx}]}"
+ fi
+ idx=$((idx + 1))
+ done
+ ) | parallel -r -j${YASHCHIKI_JOBS} 1>/dev/null
+
+ for f in "${specfiles[@]}"; do
+ echo "${f}"
+ done
+}
+
# Usage: install_from_buildcache PACKAGE...
#
# Install the given set of packages from yashchiki's buildcache.
install_from_buildcache() {
local install_failed=0
(
- _install_from_buildcache "${@}"
+ specfiles=$(_get_specfiles_for_packages_to_install "${@}")
+ install_from_buildcache_from_specfiles "${specfiles}"
) || install_failed=1
if (( install_failed == 1 )); then
@@ -306,7 +340,20 @@ install_from_buildcache() {
fi
}
-_install_from_buildcache() {
+install_environment_from_buildcache() {
+ local install_failed=0
+ (
+ specfiles=$(_get_specfiles_for_environments_to_install "${@}")
+ install_from_buildcache_from_specfiles "${specfiles}"
+ ) || install_failed=1
+
+ if (( install_failed == 1 )); then
+ echo "Error during builcache install!" >&2
+ exit 1
+ fi
+}
+
+_get_specfiles_for_packages_to_install() {
# only extract the hashes present in buildcache on first invocation
size_file_hashes=0
if [ -e "${FILE_HASHES_BUILDCACHE}" ]; then
@@ -319,10 +366,35 @@ _install_from_buildcache() {
local specfiles=()
local packages_to_install=("${@}")
readarray -t specfiles < <(get_specfiles "${packages_to_install[@]}")
+ echo "${specfiles[@]}"
+}
+
+_get_specfiles_for_environments_to_install() {
+ # only extract the hashes present in buildcache on first invocation
+ size_file_hashes=0
+ if [ -e "${FILE_HASHES_BUILDCACHE}" ]; then
+ size_file_hashes="$(wc -l <"${FILE_HASHES_BUILDCACHE}")"
+ fi
+ if (( "${size_file_hashes}" == 0 )); then
+ compute_hashes_buildcache
+ fi
+
+ local specfiles=()
+ local environments_to_install=("${@}")
+ readarray -t specfiles < <(get_specfiles_for_environments "${environments_to_install[@]}")
+ echo "${specfiles[@]}"
+}
+
+# Usage: install_from_buildcache_from_specfiles SPECFILES…
+#
+# Install the packages specified in the specfiles from yashchiki's buildcache.
+install_from_buildcache_from_specfiles() {
+ local specfiles=("${@}")
# check again that specfiles are not empty - otherwise a concretization failed
echo "DEBUG: Checking specfiles for ${packages_to_install[*]}" >&2
for spec in "${specfiles[@]}"; do
+ # FIXME: check if file exists?
if (( $(wc -l <"${spec}") == 0 )); then
echo "One of the following specs failed to concretize: " \
"${packages_to_install[@]}" >&2
diff --git a/lib/yashchiki/fetch.sh b/lib/yashchiki/fetch.sh
index 9c6c7fe1f1bb4ab475d17c66a2f44e34ad5191f9..1fb4f094a64be00fe9fc9bd9be18801f4f1ca668 100755
--- a/lib/yashchiki/fetch.sh
+++ b/lib/yashchiki/fetch.sh
@@ -124,6 +124,23 @@ done
# wait for all spawned jobs to complete
wait
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+ echo "Using spack environment"
+ ${MY_SPACK_CMD} env remove -y default || true # FIXME: drop
+ ${MY_SPACK_CMD} env create default
+ # FIXME: via style config!
+ rsync -a ${SPACK_ENVIRONMENT_REPO}/. ${YASHCHIKI_SPACK_PATH}/var/spack/environments/default
+ echo "Created spack environment, now concretizing... "
+ # FIXME: track concretizer errors here:
+ (
+ # FIXME: can we hash something to get a reproducible non-static specfile name?
+ specfile="env_specfile";
+ # FIXME: style config!
+ ${MY_SPACK_CMD} -e default --test=root --fresh concretize &&
+ ${MY_SPACK_CMD} -e default spec -y > "${specfile}"
+ ) && echo "done." || (echo "FAILED."; exit 1)
+fi
+
# verify that all concretizations were successful
if (( $(cat "${tmpfiles_concretize_err[@]}" | wc -l) > 0 )); then
{
@@ -169,6 +186,15 @@ for package in "${packages_to_fetch[@]}"; do
fi
fetch_specfiles+=( "${specfile}" )
done
+
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+ # spack fetch doesn't like proper multi-doc separators (---)
+ csplit -z --prefix=env_specfile_split --suffix-format=%02d.yaml ./env_specfile.yaml /---/ '{*}'
+ for split_specfile in ./env_specfile_split*.yaml; do
+ fetch_specfiles+=( ${split_specfile} )
+ done
+fi
+
if ! ${MY_SPACK_CMD} fetch -D "${fetch_specfiles[@]/^/-f }"; then
# propagate error
exit 1
diff --git a/lib/yashchiki/install_spack_packages.sh b/lib/yashchiki/install_spack_packages.sh
index f3350d0c4f3212547e4a2c43dd94194ac8113029..c655785b1545ffbc550463fd866977e62bebff20 100755
--- a/lib/yashchiki/install_spack_packages.sh
+++ b/lib/yashchiki/install_spack_packages.sh
@@ -13,6 +13,8 @@ cd "$HOME"
install_from_buildcache "${spack_packages[@]+"${spack_packages[@]}"}"
+# FIXME: install_from_buildcache for environments!
+
echo "INSTALLING PACKAGES"
# heurisitic: let's use -j 8 typically… (and parallelize spack builds)
SPACK_INSTALL_PARALLELISM=1 # ECM (2024-09-16): disabled $(( (${YASHCHIKI_JOBS} - 8) / 8 + 1))
@@ -34,6 +36,16 @@ for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do
wait
done
+# Install stuff for environments
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+ echo "Installing: environment \"default\" (parallelism factor ${SPACK_INSTALL_PARALLELISM})" >&2
+ # FIXME: specfile not needed if concretized in fetch step
+ for i in `seq 1 ${SPACK_INSTALL_PARALLELISM}`; do
+ (nohup ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" -e default install -j $(( (${YASHCHIKI_JOBS} - ${SPACK_INSTALL_PARALLELISM}) / ${SPACK_INSTALL_PARALLELISM} + 1 )) --fresh --no-cache --show-log-on-error | sed -e "s:^:[ENV-default-${i}] :g") &
+ done
+ wait
+fi
+
# create the filesystem views (exposed via singularity --app option)
echo "CREATING VIEWS OF META PACKAGES" >&2
cd ${MY_SPACK_FOLDER}