diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh index bb532f3e93a0bcb34dd5204b66efab37f2218772..5bb6c2a37dcff0a8325a01aff9c5b91422a1982a 100755 --- a/lib/yashchiki/commons.sh +++ b/lib/yashchiki/commons.sh @@ -231,7 +231,7 @@ fi parallel_cmds() { local num_jobs local opts OPTIND OPTARG - num_jobs="$(nproc)" + num_jobs="${YASHCHIKI_JOBS}" while getopts ":j:" opts do case $opt in @@ -417,7 +417,7 @@ get_specfiles() { fi idx=$((idx + 1)) done - ) | parallel -r -j$(nproc) 1>/dev/null + ) | parallel -r -j${YASHCHIKI_JOBS} 1>/dev/null # TODO: DELME for sf in "${specfiles[@]}"; do @@ -483,7 +483,7 @@ _install_from_buildcache() { # get all top-level directories that have to be created so that each tar process only creates its own directory local toplevel_dirs - mapfile -t toplevel_dirs < <(parallel -j "$(nproc)" \ + mapfile -t toplevel_dirs < <(parallel -j "${YASHCHIKI_JOBS}" \ "bash -c 'tar Ptf ${BUILD_CACHE_INSIDE}/{}.tar.gz | head -n 1'" < "${FILE_HASHES_TO_INSTALL_FROM_BUILDCACHE}" \ | xargs -r dirname | sort | uniq ) @@ -492,7 +492,7 @@ _install_from_buildcache() { [ ! -d "${dir}" ] && mkdir -p "${dir}" done - parallel -v -j $(nproc) tar Pxf "${BUILD_CACHE_INSIDE}/{}.tar.gz" \ + parallel -v -j ${YASHCHIKI_JOBS} tar Pxf "${BUILD_CACHE_INSIDE}/{}.tar.gz" \ < "${FILE_HASHES_TO_INSTALL_FROM_BUILDCACHE}" # have spack reindex its install contents to find the new packages diff --git a/lib/yashchiki/fetch.sh b/lib/yashchiki/fetch.sh index 289c9c1ff0506a9e6924aeba6712f8e0c6c0d594..7a4db2dff9b21922bef6bbd7fef43ad6b30a0a56 100755 --- a/lib/yashchiki/fetch.sh +++ b/lib/yashchiki/fetch.sh @@ -83,7 +83,7 @@ for package in "${packages_to_fetch[@]}"; do echo "Concretizing ${package:0:30} for fetching.." >&2 # pause if we have sufficient concretizing jobs set +x # do not clobber build log so much - while (( $(jobs | wc -l) >= $(nproc) )); do + while (( $(jobs | wc -l) >= ${YASHCHIKI_JOBS} )); do # call jobs because otherwise we will not exit the loop jobs &>/dev/null sleep 1 diff --git a/lib/yashchiki/update_build_cache_in_container.sh b/lib/yashchiki/update_build_cache_in_container.sh index 2371d59f4dee9dd088ac1a9a49916c5b66286b20..c16ee96f021b0f7b48738cd11a53b8d36ca068d3 100755 --- a/lib/yashchiki/update_build_cache_in_container.sh +++ b/lib/yashchiki/update_build_cache_in_container.sh @@ -86,9 +86,9 @@ fi cd ${MY_SPACK_FOLDER} get_hashes_to_store \ - | parallel -r ${args_progress} -j$(nproc) \ + | parallel -r ${args_progress} -j${YASHCHIKI_JOBS} \ tar Pcfz "${destination_folder}/{}.tar.gz" \"\$\(spack location -i /{}\)\" # verify integrity (of actual files, not possible symlinks) find "${destination_folder}" -type f -name "*.tar.gz" -print0 \ - | parallel -r -0 -j$(nproc) "tar Ptf '{}' 1>/dev/null" + | parallel -r -0 -j${YASHCHIKI_JOBS} "tar Ptf '{}' 1>/dev/null" diff --git a/share/yashchiki/styles/asic/create_recipe.sh b/share/yashchiki/styles/asic/create_recipe.sh index c599df7b244f8c8aa0266f2575640e8d1a81b822..6b218e11ea607653adf4e445611372bbe15fe853 100755 --- a/share/yashchiki/styles/asic/create_recipe.sh +++ b/share/yashchiki/styles/asic/create_recipe.sh @@ -236,6 +236,7 @@ From: ${DOCKER_BASE_IMAGE} export YASHCHIKI_BUILD_SPACK_GCC="${YASHCHIKI_BUILD_SPACK_GCC}" export YASHCHIKI_SPACK_GCC="${YASHCHIKI_SPACK_GCC}" export YASHCHIKI_SPACK_GCC_VERSION="${YASHCHIKI_SPACK_GCC_VERSION}" + export YASHCHIKI_JOBS="${YASHCHIKI_JOBS}" export YASHCHIKI_SPACK_CONFIG="/tmp/spack_config" export CONTAINER_STYLE="${CONTAINER_STYLE}" "${SPACK_INSTALL_SCRIPTS}/complete_spack_install_routine_called_in_post_as_root.sh" diff --git a/share/yashchiki/styles/visionary/create_recipe.sh b/share/yashchiki/styles/visionary/create_recipe.sh index a1bae05e658a62339df3a95273f38ae3d6f01de2..b6c4353a7bdfb61c36a8a61bc4f68d47a8998f5c 100755 --- a/share/yashchiki/styles/visionary/create_recipe.sh +++ b/share/yashchiki/styles/visionary/create_recipe.sh @@ -77,11 +77,12 @@ From: ${DOCKER_BASE_IMAGE} export YASHCHIKI_BUILD_SPACK_GCC="${YASHCHIKI_BUILD_SPACK_GCC}" export YASHCHIKI_SPACK_GCC="${YASHCHIKI_SPACK_GCC}" export YASHCHIKI_SPACK_GCC_VERSION="${YASHCHIKI_SPACK_GCC_VERSION}" + export YASHCHIKI_JOBS="${YASHCHIKI_JOBS}" export YASHCHIKI_SPACK_CONFIG="/tmp/spack_config" export YASHCHIKI_CACHES_ROOT="${YASHCHIKI_CACHES_ROOT}" export CONTAINER_STYLE="${CONTAINER_STYLE}" # Improve efficiency by installing system packages in the background (even - # though we set the number of worker to \$(nproc), often times - e.g. when + # though we set the number of worker to \${YASHCHIKI_JOBS}, often times - e.g. when # concretizing - only one process will be active.) # NOTE: For this to work all spack-related dependencies need to be # specified under "Inlucde:" above. install_system_dependencies.sh should