diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh
index 532c3d7255cbc6f9c6cec419a9e1d774ef7f81db..0ec605c087f9bffbb5140847467fa2dac346bdd3 100755
--- a/lib/yashchiki/commons.sh
+++ b/lib/yashchiki/commons.sh
@@ -220,37 +220,12 @@ remove_tmp_files() {
add_cleanup_step remove_tmp_files
-# get hashes in buildcache [<build_cache-directory>]
-# <buildcache-directory> defaults to ${BUILD_CACHE_INSIDE} if not supplied.
-get_hashes_in_buildcache() {
- local buildcache_dir
- buildcache_dir="${1:-${BUILD_CACHE_INSIDE}}"
-
- local resultsfile
- resultsfile=$(mktemp)
-
- if [ -d "${buildcache_dir}" ]; then
- # Naming scheme in the build_cache is <checksum>.tar.gz -> extract from full path
- ( find "${buildcache_dir}" -name "*.tar.gz" -mindepth 1 -maxdepth 1 -print0 \
- | xargs -r -0 -n 1 basename \
- | sed -e "s:\.tar\.gz$::g" \
- | sort >"${resultsfile}") || /bin/true
- fi
- echo "DEBUG: Found $(wc -l <"${resultsfile}") hashes in buildcache: ${buildcache_dir}" >&2
- cat "${resultsfile}"
- rm "${resultsfile}"
-}
-
-
-get_hashes_in_spack() {
- # we only return hashes that are actually IN spack, i.e., that reside under /opt/spack/opt/spack
- ${MY_SPACK_CMD} find --no-groups -Lp | awk '$3 ~ /^\/opt\/spack\/opt\/spack\// { print $1 }' | sort
-}
+source ${SOURCE_DIR}/get_hashes_in_buildcache.sh
compute_hashes_buildcache() {
# extract all available package hashes from buildcache
- get_hashes_in_buildcache >"${FILE_HASHES_BUILDCACHE}"
+ get_hashes_in_buildcache ${BUILD_CACHE_INSIDE} >"${FILE_HASHES_BUILDCACHE}"
}
diff --git a/lib/yashchiki/get_hashes_in_buildcache.sh b/lib/yashchiki/get_hashes_in_buildcache.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ebffb0d25b0a06574fd86c2344a46fd1bdb8b38e
--- /dev/null
+++ b/lib/yashchiki/get_hashes_in_buildcache.sh
@@ -0,0 +1,20 @@
+# get hashes in buildcache [<build_cache-directory>]
+# <buildcache-directory> defaults to ${BUILD_CACHE_INSIDE} if not supplied.
+get_hashes_in_buildcache() {
+ local buildcache_dir
+ buildcache_dir="${1}"
+
+ local resultsfile
+ resultsfile=$(mktemp)
+
+ if [ -d "${buildcache_dir}" ]; then
+ # Naming scheme in the build_cache is <checksum>.tar.gz -> extract from full path
+ ( find "${buildcache_dir}" -name "*.tar.gz" -mindepth 1 -maxdepth 1 -print0 \
+ | xargs -r -0 -n 1 basename \
+ | sed -e "s:\.tar\.gz$::g" \
+ | sort >"${resultsfile}") || /bin/true
+ fi
+ echo "DEBUG: Found $(wc -l <"${resultsfile}") hashes in buildcache: ${buildcache_dir}" >&2
+ cat "${resultsfile}"
+ rm "${resultsfile}"
+}
diff --git a/lib/yashchiki/preserve_built_spack_packages.sh b/lib/yashchiki/preserve_built_spack_packages.sh
index 5b87548527acdbacb9151f175def59b7b042c34e..b1866294b31a0d026a5f4f3ebe132ab76f3c06e4 100755
--- a/lib/yashchiki/preserve_built_spack_packages.sh
+++ b/lib/yashchiki/preserve_built_spack_packages.sh
@@ -37,7 +37,7 @@ done
mkdir -p "${target_folder}"
-"${sourcedir}/update_build_cache_in_container.sh" -d "${target_folder}" -q || /bin/true # do not fail!
+"${sourcedir}/update_build_cache_in_container.sh" -d "${target_folder}" -q -j ${YASHCHIKI_JOBS} || /bin/true # do not fail!
# preserve the specs that were concretized
pushd "${SPEC_FOLDER_IN_CONTAINER}"
diff --git a/lib/yashchiki/update_build_cache.sh b/lib/yashchiki/update_build_cache.sh
index 49ff245877e418c00b9ba7550189ffbf63dfd709..b5ebbd8c830ef01d41dc5a44bf2e3785077d2ef7 100755
--- a/lib/yashchiki/update_build_cache.sh
+++ b/lib/yashchiki/update_build_cache.sh
@@ -32,4 +32,4 @@ set +e
sudo -E singularity exec\
-B "${BUILD_CACHE_OUTSIDE}:${BUILD_CACHE_INSIDE}:rw"\
"${IMAGE_NAME}" \
- sudo -Eu spack /opt/spack_install_scripts/update_build_cache_in_container.sh -q || exit 0
+ sudo -Eu spack /opt/spack_install_scripts/update_build_cache_in_container.sh -j ${YASHCHIKI_JOBS} -q || exit 0
diff --git a/lib/yashchiki/update_build_cache_in_container.sh b/lib/yashchiki/update_build_cache_in_container.sh
index 9615bdbf3df2619397f1fe68eebf409bd36fe7a2..75875485e832c525a652a13bd23a3213b42295bb 100755
--- a/lib/yashchiki/update_build_cache_in_container.sh
+++ b/lib/yashchiki/update_build_cache_in_container.sh
@@ -29,8 +29,9 @@ destination_folder=""
destination_folder_specified=0
quiet=0
+num_jobs=1
-while getopts ":b:d:q" opts; do
+while getopts ":b:d:qj:" opts; do
case "${opts}" in
b)
base_buildcache="${OPTARG}"
@@ -42,6 +43,9 @@ while getopts ":b:d:q" opts; do
q)
quiet=1
;;
+ j)
+ num_jobs="${OPTARG}"
+ ;;
*)
usage
;;
@@ -49,7 +53,7 @@ while getopts ":b:d:q" opts; do
done
SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-source "${SOURCE_DIR}/commons.sh"
+source "${SOURCE_DIR}/get_hashes_in_buildcache.sh"
if [ -z "${base_buildcache}" ]; then
base_buildcache="${BUILD_CACHE_INSIDE}"
@@ -63,6 +67,12 @@ fi
source /opt/spack/share/spack/setup-env.sh
export SPACK_SHELL="bash"
+get_hashes_in_spack() {
+ # we only return hashes that are actually IN spack, i.e., that reside under /opt/spack/opt/spack
+ spack find --no-groups -Lp | awk '$3 ~ /^\/opt\/spack\/opt\/spack\// { print $1 }' | sort
+}
+
+
# we store all hashes currently installed that are not already in the buildcache
get_hashes_to_store() {
comm -13 <(get_hashes_in_buildcache "${base_buildcache}") <(get_hashes_in_spack)
@@ -74,7 +84,7 @@ if (( quiet == 1 )); then
fi
# find requires current working directory to be readable by spack user
-cd ${MY_SPACK_FOLDER}
+cd ${destination_folder}
# Create tmpdir in destination folder to compress into,
# atomically move compressed files into the destination folder afterwards
@@ -86,13 +96,13 @@ rm_tmpdir() {
trap rm_tmpdir EXIT
get_hashes_to_store \
- | parallel -r ${args_progress} -j${YASHCHIKI_JOBS} \
+ | parallel -r ${args_progress} -j${num_jobs} \
tar Pcfz "${tmpdir_in_destination_folder}/{}.tar.gz" \"\$\(spack location -i /{}\)\"
# verify integrity (of actual files, not possible symlinks)
find "${tmpdir_in_destination_folder}" -type f -name "*.tar.gz" -print0 \
- | parallel -r -0 -j${YASHCHIKI_JOBS} "tar Ptf '{}' 1>/dev/null"
+ | parallel -r -0 -j${num_jobs} "tar Ptf '{}' 1>/dev/null"
# atomically move files into destination folder
find "${tmpdir_in_destination_folder}" -type f -name "*.tar.gz" -print0 \
- | parallel -r -0 -j${YASHCHIKI_JOBS} "mv '{}' ${destination_folder} 1>/dev/null"
+ | parallel -r -0 -j${num_jobs} "mv '{}' ${destination_folder} 1>/dev/null"
diff --git a/share/yashchiki/utils/dump_cache.sh b/share/yashchiki/utils/dump_cache.sh
index a66a6ac8ccb7fb2676bdecaa363c7c53bf516c2b..685fc9d30da909f1bf01aadcc7167844a550ee7b 100755
--- a/share/yashchiki/utils/dump_cache.sh
+++ b/share/yashchiki/utils/dump_cache.sh
@@ -22,6 +22,7 @@ Options:
buildcache will not be dumped.
-c <container> Path to container which to dump.
-d <destination> Directory into which to dump the packages.
+ -j <num_jobs> Number of parallel jobs to use
-v Display script version.
EOF
}
@@ -30,7 +31,9 @@ EOF
# Handle command line arguments
#-----------------------------------------------------------------------
-while getopts ":hvb:c:d:" opt
+num_jobs=1
+
+while getopts ":hvb:c:d:j:" opt
do
case $opt in
@@ -44,6 +47,8 @@ do
d ) destination="${OPTARG}" ;;
+ j ) num_jobs="${OPTARG}" ;;
+
* ) echo -e "\n Option does not exist : $OPTARG\n"
usage; exit 1 ;;
@@ -85,6 +90,7 @@ args+=(
"${container}"
"/opt/spack_install_scripts/update_build_cache_in_container.sh"
"-d" "/opt/dumptarget"
+ "-j" "${num_jobs}"
)
if [ -n "${buildcache:-}" ]; then