From 721b828e6937390040b200107371d57b2e85e5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Wed, 24 Jul 2024 17:10:43 +0200
Subject: [PATCH] feat: parallelize spack install (i.e. build) of uncached
packages
Change-Id: Ibb9cb13002dcfe159e93cd1676cb7d5e142ff328
---
bin/yashchiki | 2 +-
lib/yashchiki/install_spack_packages.sh | 10 ++++++++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/bin/yashchiki b/bin/yashchiki
index 71eca0ac..296ecc0c 100755
--- a/bin/yashchiki
+++ b/bin/yashchiki
@@ -120,7 +120,7 @@ parser.add_argument(
# optional options
parser.add_argument(
"-j", "--jobs", type=int, default=1,
- help="Number of parallel jobs to use when concurt execution of tasks is available.")
+ help="Number of parallel jobs to use when concurrent execution of tasks is available.")
parser.add_argument(
"--debug", action="store_true",
help="Enable debug-level logging.")
diff --git a/lib/yashchiki/install_spack_packages.sh b/lib/yashchiki/install_spack_packages.sh
index 2a98731d..f3350d0c 100755
--- a/lib/yashchiki/install_spack_packages.sh
+++ b/lib/yashchiki/install_spack_packages.sh
@@ -14,6 +14,8 @@ cd "$HOME"
install_from_buildcache "${spack_packages[@]+"${spack_packages[@]}"}"
echo "INSTALLING PACKAGES"
+# heurisitic: let's use -j 8 typically… (and parallelize spack builds)
+SPACK_INSTALL_PARALLELISM=1 # ECM (2024-09-16): disabled $(( (${YASHCHIKI_JOBS} - 8) / 8 + 1))
for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do
# Disable cache because we already installed from build cache.
# Also there is a bug that when `--no-cache` is not specified, install will
@@ -24,8 +26,12 @@ for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do
echo "ERROR: Failed to concretize ${package} for install." >&2
exit 1
fi
- echo "Installing: ${package}" >&2
- ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" install --fresh --no-cache --show-log-on-error --file "${specfile}"
+
+ echo "Installing: ${package} (parallelism factor ${SPACK_INSTALL_PARALLELISM})" >&2
+ for i in `seq 1 ${SPACK_INSTALL_PARALLELISM}`; do
+ (nohup ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" install -j $(( (${YASHCHIKI_JOBS} - ${SPACK_INSTALL_PARALLELISM}) / ${SPACK_INSTALL_PARALLELISM} + 1 )) --fresh --no-cache --show-log-on-error --file "${specfile}" | sed -e "s:^:[${package}-${i}] :g") &
+ done
+ wait
done
# create the filesystem views (exposed via singularity --app option)
--
GitLab