From 95abcc9171fed5e0a722a7b53bab9bb14eded4d2 Mon Sep 17 00:00:00 2001
From: Philipp Spilger <philipp.spilger@kip.uni-heidelberg.de>
Date: Fri, 16 Feb 2024 10:16:56 +0100
Subject: [PATCH] feat: Use --jobs at all parallel invokations
Change-Id: I639072b31bd59fe10a7ebb20c1ef7a57ac96cc57
---
lib/yashchiki/commons.sh | 8 ++++----
lib/yashchiki/fetch.sh | 2 +-
lib/yashchiki/update_build_cache_in_container.sh | 4 ++--
share/yashchiki/styles/asic/create_recipe.sh | 1 +
share/yashchiki/styles/visionary/create_recipe.sh | 3 ++-
5 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh
index bb532f3e..5bb6c2a3 100755
--- a/lib/yashchiki/commons.sh
+++ b/lib/yashchiki/commons.sh
@@ -231,7 +231,7 @@ fi
parallel_cmds() {
local num_jobs
local opts OPTIND OPTARG
- num_jobs="$(nproc)"
+ num_jobs="${YASHCHIKI_JOBS}"
while getopts ":j:" opts
do
case $opt in
@@ -417,7 +417,7 @@ get_specfiles() {
fi
idx=$((idx + 1))
done
- ) | parallel -r -j$(nproc) 1>/dev/null
+ ) | parallel -r -j${YASHCHIKI_JOBS} 1>/dev/null
# TODO: DELME
for sf in "${specfiles[@]}"; do
@@ -483,7 +483,7 @@ _install_from_buildcache() {
# get all top-level directories that have to be created so that each tar process only creates its own directory
local toplevel_dirs
- mapfile -t toplevel_dirs < <(parallel -j "$(nproc)" \
+ mapfile -t toplevel_dirs < <(parallel -j "${YASHCHIKI_JOBS}" \
"bash -c 'tar Ptf ${BUILD_CACHE_INSIDE}/{}.tar.gz | head -n 1'" < "${FILE_HASHES_TO_INSTALL_FROM_BUILDCACHE}" \
| xargs -r dirname | sort | uniq )
@@ -492,7 +492,7 @@ _install_from_buildcache() {
[ ! -d "${dir}" ] && mkdir -p "${dir}"
done
- parallel -v -j $(nproc) tar Pxf "${BUILD_CACHE_INSIDE}/{}.tar.gz" \
+ parallel -v -j ${YASHCHIKI_JOBS} tar Pxf "${BUILD_CACHE_INSIDE}/{}.tar.gz" \
< "${FILE_HASHES_TO_INSTALL_FROM_BUILDCACHE}"
# have spack reindex its install contents to find the new packages
diff --git a/lib/yashchiki/fetch.sh b/lib/yashchiki/fetch.sh
index 289c9c1f..7a4db2df 100755
--- a/lib/yashchiki/fetch.sh
+++ b/lib/yashchiki/fetch.sh
@@ -83,7 +83,7 @@ for package in "${packages_to_fetch[@]}"; do
echo "Concretizing ${package:0:30} for fetching.." >&2
# pause if we have sufficient concretizing jobs
set +x # do not clobber build log so much
- while (( $(jobs | wc -l) >= $(nproc) )); do
+ while (( $(jobs | wc -l) >= ${YASHCHIKI_JOBS} )); do
# call jobs because otherwise we will not exit the loop
jobs &>/dev/null
sleep 1
diff --git a/lib/yashchiki/update_build_cache_in_container.sh b/lib/yashchiki/update_build_cache_in_container.sh
index 2371d59f..c16ee96f 100755
--- a/lib/yashchiki/update_build_cache_in_container.sh
+++ b/lib/yashchiki/update_build_cache_in_container.sh
@@ -86,9 +86,9 @@ fi
cd ${MY_SPACK_FOLDER}
get_hashes_to_store \
- | parallel -r ${args_progress} -j$(nproc) \
+ | parallel -r ${args_progress} -j${YASHCHIKI_JOBS} \
tar Pcfz "${destination_folder}/{}.tar.gz" \"\$\(spack location -i /{}\)\"
# verify integrity (of actual files, not possible symlinks)
find "${destination_folder}" -type f -name "*.tar.gz" -print0 \
- | parallel -r -0 -j$(nproc) "tar Ptf '{}' 1>/dev/null"
+ | parallel -r -0 -j${YASHCHIKI_JOBS} "tar Ptf '{}' 1>/dev/null"
diff --git a/share/yashchiki/styles/asic/create_recipe.sh b/share/yashchiki/styles/asic/create_recipe.sh
index c599df7b..6b218e11 100755
--- a/share/yashchiki/styles/asic/create_recipe.sh
+++ b/share/yashchiki/styles/asic/create_recipe.sh
@@ -236,6 +236,7 @@ From: ${DOCKER_BASE_IMAGE}
export YASHCHIKI_BUILD_SPACK_GCC="${YASHCHIKI_BUILD_SPACK_GCC}"
export YASHCHIKI_SPACK_GCC="${YASHCHIKI_SPACK_GCC}"
export YASHCHIKI_SPACK_GCC_VERSION="${YASHCHIKI_SPACK_GCC_VERSION}"
+ export YASHCHIKI_JOBS="${YASHCHIKI_JOBS}"
export YASHCHIKI_SPACK_CONFIG="/tmp/spack_config"
export CONTAINER_STYLE="${CONTAINER_STYLE}"
"${SPACK_INSTALL_SCRIPTS}/complete_spack_install_routine_called_in_post_as_root.sh"
diff --git a/share/yashchiki/styles/visionary/create_recipe.sh b/share/yashchiki/styles/visionary/create_recipe.sh
index a1bae05e..b6c4353a 100755
--- a/share/yashchiki/styles/visionary/create_recipe.sh
+++ b/share/yashchiki/styles/visionary/create_recipe.sh
@@ -77,11 +77,12 @@ From: ${DOCKER_BASE_IMAGE}
export YASHCHIKI_BUILD_SPACK_GCC="${YASHCHIKI_BUILD_SPACK_GCC}"
export YASHCHIKI_SPACK_GCC="${YASHCHIKI_SPACK_GCC}"
export YASHCHIKI_SPACK_GCC_VERSION="${YASHCHIKI_SPACK_GCC_VERSION}"
+ export YASHCHIKI_JOBS="${YASHCHIKI_JOBS}"
export YASHCHIKI_SPACK_CONFIG="/tmp/spack_config"
export YASHCHIKI_CACHES_ROOT="${YASHCHIKI_CACHES_ROOT}"
export CONTAINER_STYLE="${CONTAINER_STYLE}"
# Improve efficiency by installing system packages in the background (even
- # though we set the number of worker to \$(nproc), often times - e.g. when
+ # though we set the number of worker to \${YASHCHIKI_JOBS}, often times - e.g. when
# concretizing - only one process will be active.)
# NOTE: For this to work all spack-related dependencies need to be
# specified under "Inlucde:" above. install_system_dependencies.sh should
--
GitLab