diff --git a/bin/yashchiki b/bin/yashchiki index 71eca0acdf96158cd71b7c3cf96ebd78a8c739bb..296ecc0c33c62aa5862e96c00ff56d18455075a4 100755 --- a/bin/yashchiki +++ b/bin/yashchiki @@ -120,7 +120,7 @@ parser.add_argument( # optional options parser.add_argument( "-j", "--jobs", type=int, default=1, - help="Number of parallel jobs to use when concurt execution of tasks is available.") + help="Number of parallel jobs to use when concurrent execution of tasks is available.") parser.add_argument( "--debug", action="store_true", help="Enable debug-level logging.") diff --git a/lib/yashchiki/install_spack_packages.sh b/lib/yashchiki/install_spack_packages.sh index 2a98731d6ef6782028a2e954f76e54ece872b6cd..f3350d0c4f3212547e4a2c43dd94194ac8113029 100755 --- a/lib/yashchiki/install_spack_packages.sh +++ b/lib/yashchiki/install_spack_packages.sh @@ -14,6 +14,8 @@ cd "$HOME" install_from_buildcache "${spack_packages[@]+"${spack_packages[@]}"}" echo "INSTALLING PACKAGES" +# heurisitic: let's use -j 8 typically… (and parallelize spack builds) +SPACK_INSTALL_PARALLELISM=1 # ECM (2024-09-16): disabled $(( (${YASHCHIKI_JOBS} - 8) / 8 + 1)) for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do # Disable cache because we already installed from build cache. # Also there is a bug that when `--no-cache` is not specified, install will @@ -24,8 +26,12 @@ for package in "${spack_packages[@]+"${spack_packages[@]}"}"; do echo "ERROR: Failed to concretize ${package} for install." >&2 exit 1 fi - echo "Installing: ${package}" >&2 - ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" install --fresh --no-cache --show-log-on-error --file "${specfile}" + + echo "Installing: ${package} (parallelism factor ${SPACK_INSTALL_PARALLELISM})" >&2 + for i in `seq 1 ${SPACK_INSTALL_PARALLELISM}`; do + (nohup ${MY_SPACK_CMD} "${SPACK_ARGS_INSTALL[@]+"${SPACK_ARGS_INSTALL[@]}"}" install -j $(( (${YASHCHIKI_JOBS} - ${SPACK_INSTALL_PARALLELISM}) / ${SPACK_INSTALL_PARALLELISM} + 1 )) --fresh --no-cache --show-log-on-error --file "${specfile}" | sed -e "s:^:[${package}-${i}] :g") & + done + wait done # create the filesystem views (exposed via singularity --app option)