feat: support spack environments

Change-Id: I8ebab1ca1e179ab7b50c233d54407d7bb4d02056

bin/yashchiki

@@ -167,6 +167,7 @@ env = os.environ.copy()
 env.update({
     "DOCKER_BASE_IMAGE": config["docker_base_image"],
     "BUILD_BASE_SANDBOX": str(int(config["build_base_sandbox"])),
+    "SPACK_ENVIRONMENT": str(int(config["spack_environment"])),
     # This needs to be here because otherwise the default python
     # (2.7.18) will pollute the spec and lead to a conflict
     # can be removed as soon as the explicit preferred version

lib/yashchiki/commons.sh

@@ -237,6 +237,15 @@ get_specfile_name() {
                               awk '{ print "spec_" $1 ".yaml" }')"
 }
 
+# Get the name of given environment, should only be called if one does not
+# depend on the specfile existing or having correct content, see
+# get_specfiles().
+get_specfile_name_for_environment() {
+    echo -n "${SPEC_FOLDER}/$(echo "environment_$1" | sha256sum |
+                              awk '{ print "spec_" $1 ".yaml" }')"
+}
+
+
 # Compute the concrete specfile for the given packages.
 #
 # Spec files are only computed once and afterwards their names are emitted via
@@ -291,13 +300,38 @@ get_specfiles() {
     done
 }
 
+# cf. get_specfiles but for spack environments
+get_specfiles_for_environments() {
+    local specfiles=()
+
+    for environment in "${@}"; do
+        # compute spec and put into temporary file derived from package name
+        specfiles+=("$(get_specfile_name_for_environment "${environment}")")
+    done
+
+    (
+    local idx=0
+    for package in "${@}"; do
+        if [ ! -f "${specfiles[${idx}]}" ]; then
+            echo "${MY_SPACK_CMD} -e \"${environment}\" spec -y > ${specfiles[${idx}]}"
+        fi
+        idx=$((idx + 1))
+    done
+    ) | parallel -r -j${YASHCHIKI_JOBS} 1>/dev/null
+
+    for f in "${specfiles[@]}"; do
+        echo "${f}"
+    done
+}
+
 # Usage: install_from_buildcache PACKAGE...
 #
 # Install the given set of packages from yashchiki's buildcache.
 install_from_buildcache() {
     local install_failed=0
     (
-        _install_from_buildcache "${@}"
+        specfiles=$(_get_specfiles_for_packages_to_install "${@}")
+        install_from_buildcache_from_specfiles "${specfiles}"
     ) || install_failed=1
 
     if (( install_failed == 1 )); then
@@ -306,7 +340,20 @@ install_from_buildcache() {
     fi
 }
 
-_install_from_buildcache() {
+install_environment_from_buildcache() {
+    local install_failed=0
+    (
+        specfiles=$(_get_specfiles_for_environments_to_install "${@}")
+        install_from_buildcache_from_specfiles "${specfiles}"
+    ) || install_failed=1
+
+    if (( install_failed == 1 )); then
+        echo "Error during builcache install!" >&2
+        exit 1
+    fi
+}
+
+_get_specfiles_for_packages_to_install() {
     # only extract the hashes present in buildcache on first invocation
     size_file_hashes=0
     if [ -e "${FILE_HASHES_BUILDCACHE}" ]; then
@@ -319,10 +366,35 @@ _install_from_buildcache() {
     local specfiles=()
     local packages_to_install=("${@}")
     readarray -t specfiles < <(get_specfiles "${packages_to_install[@]}")
+    echo "${specfiles[@]}"
+}
+
+_get_specfiles_for_environments_to_install() {
+    # only extract the hashes present in buildcache on first invocation
+    size_file_hashes=0
+    if [ -e "${FILE_HASHES_BUILDCACHE}" ]; then
+        size_file_hashes="$(wc -l <"${FILE_HASHES_BUILDCACHE}")"
+    fi
+    if (( "${size_file_hashes}" == 0 )); then
+        compute_hashes_buildcache
+    fi
+
+    local specfiles=()
+    local environments_to_install=("${@}")
+    readarray -t specfiles < <(get_specfiles_for_environments "${environments_to_install[@]}")
+    echo "${specfiles[@]}"
+}
+
+# Usage: install_from_buildcache_from_specfiles SPECFILES…
+#
+# Install the packages specified in the specfiles from yashchiki's buildcache.
+install_from_buildcache_from_specfiles() {
+    local specfiles=("${@}")
 
     # check again that specfiles are not empty - otherwise a concretization failed
     echo "DEBUG: Checking specfiles for ${packages_to_install[*]}" >&2
     for spec in "${specfiles[@]}"; do
+        # FIXME: check if file exists?
         if (( $(wc -l <"${spec}") == 0 )); then
             echo "One of the following specs failed to concretize: " \
                  "${packages_to_install[@]}" >&2

lib/yashchiki/fetch.sh

@@ -124,6 +124,23 @@ done
 # wait for all spawned jobs to complete
 wait
 
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+    echo "Using spack environment"
+    ${MY_SPACK_CMD} env remove -y default || true # FIXME: drop
+    ${MY_SPACK_CMD} env create default
+    # FIXME: via style config!
+    rsync -a ${SPACK_ENVIRONMENT_REPO}/. ${YASHCHIKI_SPACK_PATH}/var/spack/environments/default
+    echo "Created spack environment, now concretizing... "
+    # FIXME: track concretizer errors here:
+    (
+        # FIXME: can we hash something to get a reproducible non-static specfile name?
+        specfile="env_specfile";
+        # FIXME: style config!
+        ${MY_SPACK_CMD} -e default --test=root --fresh concretize &&
+        ${MY_SPACK_CMD} -e default spec -y > "${specfile}"
+    ) && echo "done." || (echo "FAILED."; exit 1)
+fi
+
 # verify that all concretizations were successful
 if (( $(cat "${tmpfiles_concretize_err[@]}" | wc -l) > 0 )); then
     {
@@ -169,6 +186,15 @@ for package in "${packages_to_fetch[@]}"; do
     fi
     fetch_specfiles+=( "${specfile}" )
 done
+
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+    # spack fetch doesn't like proper multi-doc separators (---)
+    csplit -z --prefix=env_specfile_split --suffix-format=%02d.yaml ./env_specfile.yaml /---/ '{*}'
+    for split_specfile in ./env_specfile_split*.yaml; do
+        fetch_specfiles+=( ${split_specfile} )
+    done
+fi
+
 if ! ${MY_SPACK_CMD} fetch -D "${fetch_specfiles[@]/^/-f }"; then
     # propagate error
     exit 1

lib/yashchiki/install_spack_packages.sh

@@ -13,6 +13,8 @@ cd "$HOME"
 
 install_from_buildcache "${spack_packages[@]+"${spack_packages[@]}"}"
 
+# FIXME: install_from_buildcache for environments!
+
 echo "INSTALLING PACKAGES"
 # heurisitic: let's use -j 8 typically… (and parallelize spack builds)
 SPACK_INSTALL_PARALLELISM=1 # ECM (2024-09-16): disabled $(( (${YASHCHIKI_JOBS} - 8) / 8 + 1))
@@ -34,6 +36,16 @@ for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do
     wait
 done
 
+# Install stuff for environments
+if [ -n "${SPACK_ENVIRONMENT:-}" ]; then
+    echo "Installing: environment \"default\" (parallelism factor ${SPACK_INSTALL_PARALLELISM})" >&2
+    # FIXME: specfile not needed if concretized in fetch step
+    for i in `seq 1 ${SPACK_INSTALL_PARALLELISM}`; do
+        (nohup ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" -e default install -j $(( (${YASHCHIKI_JOBS} - ${SPACK_INSTALL_PARALLELISM}) / ${SPACK_INSTALL_PARALLELISM} + 1 )) --fresh --no-cache --show-log-on-error | sed -e "s:^:[ENV-default-${i}] :g") &
+    done
+    wait
+fi
+
 # create the filesystem views (exposed via singularity --app option)
 echo "CREATING VIEWS OF META PACKAGES" >&2
 cd ${MY_SPACK_FOLDER}