From 03b926cead5b35efa6d4e8042c40a7df400940a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Thu, 29 Jul 2021 13:58:38 +0200
Subject: [PATCH] Fix: Allow for empty build/source cache and output
`parallel` and `xargs` both support a `-r` option to avoid
running a command if the input is empty ("")
Change-Id: Ib333b8f89874fb1e76c58d887e2c15c949de0907
---
.ci/commons.sh | 10 +++++-----
.ci/fetch.sh | 2 +-
.ci/generate_modules.sh | 2 +-
.ci/update_build_cache_in_container.sh | 4 ++--
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/.ci/commons.sh b/.ci/commons.sh
index 104e893f..1ff5119f 100755
--- a/.ci/commons.sh
+++ b/.ci/commons.sh
@@ -336,7 +336,7 @@ parallel_cmds() {
done
shift $(( OPTIND - 1 ))
- grep -v "^\(#\|[[:space:]]*$\)" "${@}" | parallel -j "${num_jobs}"
+ grep -v "^\(#\|[[:space:]]*$\)" "${@}" | parallel -r -j "${num_jobs}"
}
populate_views() {
@@ -446,7 +446,7 @@ get_hashes_in_buildcache() {
if [ -d "${buildcache_dir}" ]; then
# Naming scheme in the build_cache is <checksum>.tar.gz -> extract from full path
( find "${buildcache_dir}" -name "*.tar.gz" -mindepth 1 -maxdepth 1 -print0 \
- | xargs -0 -n 1 basename \
+ | xargs -r -0 -n 1 basename \
| sed -e "s:\.tar\.gz$::g" \
| sort >"${resultsfile}") || /bin/true
fi
@@ -512,7 +512,7 @@ get_specfiles() {
fi
idx=$((idx + 1))
done
- ) | parallel -j$(nproc) 1>/dev/null
+ ) | parallel -r -j$(nproc) 1>/dev/null
# TODO: DELME
for sf in "${specfiles[@]}"; do
@@ -580,7 +580,7 @@ _install_from_buildcache() {
local toplevel_dirs
mapfile -t toplevel_dirs < <(parallel -j "$(nproc)" \
"bash -c 'tar Ptf ${BUILD_CACHE_INSIDE}/{}.tar.gz | head -n 1'" < "${FILE_HASHES_TO_INSTALL_FROM_BUILDCACHE}" \
- | xargs dirname | sort | uniq )
+ | xargs -r dirname | sort | uniq )
# ensure all toplevel directories exist
for dir in "${toplevel_dirs[@]+"${toplevel_dirs[@]}"}"; do
@@ -605,7 +605,7 @@ get_latest_failed_build_cache_name() {
possible_build_caches="$(mktemp)"
find "${BASE_BUILD_CACHE_FAILED_OUTSIDE}" -mindepth 1 -maxdepth 1 -type d -name "${change_num}*" -print0 \
- | xargs -n 1 -r -0 basename > "${possible_build_caches}"
+ | xargs -r -n 1 -r -0 basename > "${possible_build_caches}"
if (( $(wc -l <"${possible_build_caches}") == 0 )); then
return 0
diff --git a/.ci/fetch.sh b/.ci/fetch.sh
index e7527726..7e33986b 100755
--- a/.ci/fetch.sh
+++ b/.ci/fetch.sh
@@ -11,7 +11,7 @@ source "${SOURCE_DIR}/commons.sh"
# cache, so we switch from $HOME/download_cache to $HOME/source_cache
mkdir -p "${PWD}/spack/var/spack/cache/"
find "${SOURCE_CACHE_DIR}" -mindepth 1 -maxdepth 1 -print0 \
- | xargs -n 1 "-I{}" -0 cp -vrl '{}' "${PWD}/spack/var/spack/cache/"
+ | xargs -r -n 1 "-I{}" -0 cp -vrl '{}' "${PWD}/spack/var/spack/cache/"
# set download mirror stuff to prefill outside of container
export MY_SPACK_FOLDER="$PWD/spack"
diff --git a/.ci/generate_modules.sh b/.ci/generate_modules.sh
index 1fa44698..84a020ce 100755
--- a/.ci/generate_modules.sh
+++ b/.ci/generate_modules.sh
@@ -29,7 +29,7 @@ EOF
(echo; echo) >> "${TARGET}"
cat <<EOF >> "${TARGET}"
-source "${PATH_MODULES}/\$(readlink -f /proc/\$\$/exe | xargs basename)"
+source "${PATH_MODULES}/\$(readlink -f /proc/\$\$/exe | xargs -r basename)"
EOF
# Provide MODULESHOME for all singularity environments
diff --git a/.ci/update_build_cache_in_container.sh b/.ci/update_build_cache_in_container.sh
index aa7061c3..38a4e421 100755
--- a/.ci/update_build_cache_in_container.sh
+++ b/.ci/update_build_cache_in_container.sh
@@ -83,9 +83,9 @@ if (( quiet == 1 )); then
fi
get_hashes_to_store \
- | parallel ${args_progress} -j$(nproc) \
+ | parallel -r ${args_progress} -j$(nproc) \
tar Pcfz "${destination_folder}/{}.tar.gz" \"\$\(spack location -i /{}\)\"
# verify integrity (of actual files, not possible symlinks)
find "${destination_folder}" -type f -name "*.tar.gz" -print0 \
- | parallel -0 -j$(nproc) "tar Ptf '{}' 1>/dev/null"
+ | parallel -r -0 -j$(nproc) "tar Ptf '{}' 1>/dev/null"
--
GitLab