fix: dump_cache

Change-Id: Iedfb185b88a8c142b0a0ded0136bf06df806bfcf

lib/yashchiki/commons.sh

@@ -220,37 +220,12 @@ remove_tmp_files() {
 add_cleanup_step remove_tmp_files
 
 
-# get hashes in buildcache [<build_cache-directory>]
-# <buildcache-directory> defaults to ${BUILD_CACHE_INSIDE} if not supplied.
-get_hashes_in_buildcache() {
-    local buildcache_dir
-    buildcache_dir="${1:-${BUILD_CACHE_INSIDE}}"
-
-    local resultsfile
-    resultsfile=$(mktemp)
-
-    if [ -d "${buildcache_dir}" ]; then
-        # Naming scheme in the build_cache is <checksum>.tar.gz -> extract from full path
-        ( find "${buildcache_dir}" -name "*.tar.gz" -mindepth 1 -maxdepth 1 -print0 \
-            | xargs -r -0 -n 1 basename \
-            | sed -e "s:\.tar\.gz$::g" \
-	    | sort >"${resultsfile}") || /bin/true
-    fi
-    echo "DEBUG: Found $(wc -l <"${resultsfile}") hashes in buildcache: ${buildcache_dir}" >&2
-    cat "${resultsfile}"
-    rm "${resultsfile}"
-}
-
-
-get_hashes_in_spack() {
-    # we only return hashes that are actually IN spack, i.e., that reside under /opt/spack/opt/spack
-    ${MY_SPACK_CMD} find --no-groups -Lp | awk '$3 ~ /^\/opt\/spack\/opt\/spack\// { print $1 }' | sort
-}
+source ${SOURCE_DIR}/get_hashes_in_buildcache.sh
 
 
 compute_hashes_buildcache() {
     # extract all available package hashes from buildcache
-    get_hashes_in_buildcache >"${FILE_HASHES_BUILDCACHE}"
+    get_hashes_in_buildcache ${BUILD_CACHE_INSIDE} >"${FILE_HASHES_BUILDCACHE}"
 }
 
 

lib/yashchiki/get_hashes_in_buildcache.sh

+# get hashes in buildcache [<build_cache-directory>]
+# <buildcache-directory> defaults to ${BUILD_CACHE_INSIDE} if not supplied.
+get_hashes_in_buildcache() {
+    local buildcache_dir
+    buildcache_dir="${1}"
+
+    local resultsfile
+    resultsfile=$(mktemp)
+
+    if [ -d "${buildcache_dir}" ]; then
+        # Naming scheme in the build_cache is <checksum>.tar.gz -> extract from full path
+        ( find "${buildcache_dir}" -name "*.tar.gz" -mindepth 1 -maxdepth 1 -print0 \
+            | xargs -r -0 -n 1 basename \
+            | sed -e "s:\.tar\.gz$::g" \
+	    | sort >"${resultsfile}") || /bin/true
+    fi
+    echo "DEBUG: Found $(wc -l <"${resultsfile}") hashes in buildcache: ${buildcache_dir}" >&2
+    cat "${resultsfile}"
+    rm "${resultsfile}"
+}

lib/yashchiki/preserve_built_spack_packages.sh

@@ -37,7 +37,7 @@ done
 
 mkdir -p "${target_folder}"
 
-"${sourcedir}/update_build_cache_in_container.sh" -d "${target_folder}" -q || /bin/true  # do not fail!
+"${sourcedir}/update_build_cache_in_container.sh" -d "${target_folder}" -q -j ${YASHCHIKI_JOBS} || /bin/true  # do not fail!
 
 # preserve the specs that were concretized
 pushd "${SPEC_FOLDER_IN_CONTAINER}"

lib/yashchiki/update_build_cache.sh

@@ -32,4 +32,4 @@ set +e
 sudo -E singularity exec\
     -B "${BUILD_CACHE_OUTSIDE}:${BUILD_CACHE_INSIDE}:rw"\
     "${IMAGE_NAME}" \
-    sudo -Eu spack /opt/spack_install_scripts/update_build_cache_in_container.sh -q || exit 0
+    sudo -Eu spack /opt/spack_install_scripts/update_build_cache_in_container.sh -j ${YASHCHIKI_JOBS} -q || exit 0

lib/yashchiki/update_build_cache_in_container.sh

@@ -29,8 +29,9 @@ destination_folder=""
 
 destination_folder_specified=0
 quiet=0
+num_jobs=1
 
-while getopts ":b:d:q" opts; do
+while getopts ":b:d:qj:" opts; do
     case "${opts}" in
         b)
             base_buildcache="${OPTARG}"
@@ -42,6 +43,9 @@ while getopts ":b:d:q" opts; do
         q)
             quiet=1
             ;;
+        j)
+            num_jobs="${OPTARG}"
+            ;;
         *)
             usage
             ;;
@@ -49,7 +53,7 @@ while getopts ":b:d:q" opts; do
 done
 
 SOURCE_DIR="$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")"
-source "${SOURCE_DIR}/commons.sh"
+source "${SOURCE_DIR}/get_hashes_in_buildcache.sh"
 
 if [ -z "${base_buildcache}" ]; then
     base_buildcache="${BUILD_CACHE_INSIDE}"
@@ -63,6 +67,12 @@ fi
 source /opt/spack/share/spack/setup-env.sh
 export SPACK_SHELL="bash"
 
+get_hashes_in_spack() {
+    # we only return hashes that are actually IN spack, i.e., that reside under /opt/spack/opt/spack
+    spack find --no-groups -Lp | awk '$3 ~ /^\/opt\/spack\/opt\/spack\// { print $1 }' | sort
+}
+
+
 # we store all hashes currently installed that are not already in the buildcache
 get_hashes_to_store() {
     comm -13 <(get_hashes_in_buildcache "${base_buildcache}") <(get_hashes_in_spack)
@@ -74,7 +84,7 @@ if (( quiet == 1 )); then
 fi
 
 # find requires current working directory to be readable by spack user
-cd ${MY_SPACK_FOLDER}
+cd ${destination_folder}
 
 # Create tmpdir in destination folder to compress into,
 # atomically move compressed files into the destination folder afterwards
@@ -86,13 +96,13 @@ rm_tmpdir() {
 trap rm_tmpdir EXIT
 
 get_hashes_to_store \
-    | parallel -r ${args_progress} -j${YASHCHIKI_JOBS} \
+    | parallel -r ${args_progress} -j${num_jobs} \
         tar Pcfz "${tmpdir_in_destination_folder}/{}.tar.gz" \"\$\(spack location -i /{}\)\"
 
 # verify integrity (of actual files, not possible symlinks)
 find "${tmpdir_in_destination_folder}" -type f -name "*.tar.gz" -print0 \
-    | parallel -r -0 -j${YASHCHIKI_JOBS} "tar Ptf '{}' 1>/dev/null"
+    | parallel -r -0 -j${num_jobs} "tar Ptf '{}' 1>/dev/null"
 
 # atomically move files into destination folder
 find "${tmpdir_in_destination_folder}" -type f -name "*.tar.gz" -print0 \
-    | parallel -r -0 -j${YASHCHIKI_JOBS} "mv '{}' ${destination_folder} 1>/dev/null"
+    | parallel -r -0 -j${num_jobs} "mv '{}' ${destination_folder} 1>/dev/null"

share/yashchiki/utils/dump_cache.sh

@@ -22,6 +22,7 @@ Options:
                       buildcache will not be dumped.
     -c <container>    Path to container which to dump.
     -d <destination>  Directory into which to dump the packages.
+    -j <num_jobs>     Number of parallel jobs to use
     -v                Display script version.
 EOF
 }
@@ -30,7 +31,9 @@ EOF
 #  Handle command line arguments
 #-----------------------------------------------------------------------
 
-while getopts ":hvb:c:d:" opt
+num_jobs=1
+
+while getopts ":hvb:c:d:j:" opt
 do
   case $opt in
 
@@ -44,6 +47,8 @@ do
 
     d )  destination="${OPTARG}" ;;
 
+    j )  num_jobs="${OPTARG}" ;;
+
     * )  echo -e "\n  Option does not exist : $OPTARG\n"
           usage; exit 1   ;;
 
@@ -85,6 +90,7 @@ args+=(
     "${container}"
     "/opt/spack_install_scripts/update_build_cache_in_container.sh"
     "-d" "/opt/dumptarget"
+    "-j" "${num_jobs}"
 )
 
 if [ -n "${buildcache:-}" ]; then