Skip to content
Snippets Groups Projects
Commit 48541cc2 authored by Eric Müller's avatar Eric Müller :mountain_bicyclist:
Browse files

fix: omit OCI cache handling code if cache isn't specified

Changes in CI file allow to use existing spack deployments w/o cache.
parent a000bba0
No related branches found
No related tags found
2 merge requests!651create new experimental release,!627Disable OCI cache usage for jobs w/o install prefix "/esd"
...@@ -16,7 +16,7 @@ EBRAINS_REPO=$3 # location of ebrains-spack-builds repository ...@@ -16,7 +16,7 @@ EBRAINS_REPO=$3 # location of ebrains-spack-builds repository
EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated
UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional) UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional)
UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results
OCI_CACHE_PREFIX=$7 export OCI_CACHE_PREFIX=$7
# specify location of .spack dir (by default in ~) # specify location of .spack dir (by default in ~)
# this is where cache and configuration settings are stored # this is where cache and configuration settings are stored
...@@ -82,39 +82,42 @@ spack concretize --force --fresh --test root ...@@ -82,39 +82,42 @@ spack concretize --force --fresh --test root
export CACHE_SPECFILE=${CACHE_SPECFILE:-"env_specfile.yaml"} export CACHE_SPECFILE=${CACHE_SPECFILE:-"env_specfile.yaml"}
export YASHCHIKI_HOME=${EBRAINS_REPO}/vendor/yashchiki export YASHCHIKI_HOME=${EBRAINS_REPO}/vendor/yashchiki
export OCI_CACHE_PREFIX=${OCI_CACHE_PREFIX:-${HARBOR_HOST:-"docker-registry.ebrains.eu"}/${HARBOR_PROJECT:-"esd"}/master}
export SPACK_CACHE_SOURCE=${SPACK_CACHE_SOURCE:-${INSTALLATION_ROOT}/spack/var/spack/cache} export SPACK_CACHE_SOURCE=${SPACK_CACHE_SOURCE:-${INSTALLATION_ROOT}/spack/var/spack/cache}
export SPACK_CACHE_BUILD=${SPACK_CACHE_BUILD:-${INSTALLATION_ROOT}/spack/var/spack/cache} export SPACK_CACHE_BUILD=${SPACK_CACHE_BUILD:-${INSTALLATION_ROOT}/spack/var/spack/cache}
# dump dag to file # dump dag to file
spack spec -y > "${CACHE_SPECFILE}" spack spec -y > "${CACHE_SPECFILE}"
# fetch missing sources (if packages not yet installed) if [ -n "${OCI_CACHE_PREFIX}" ]; then
python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \ # fetch missing sources (if packages not yet installed)
--local-cache=${SPACK_CACHE_SOURCE} \ python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
--remote-cache-type=oci \ --local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \ --remote-cache-type=oci \
--yashchiki-home=${YASHCHIKI_HOME} \ --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
missing_paths_sources.dat ${CACHE_SPECFILE} --yashchiki-home=${YASHCHIKI_HOME} \
# fetch missing build results (if packages not yet installed) missing_paths_sources.dat ${CACHE_SPECFILE}
python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \ # fetch missing build results (if packages not yet installed)
--local-cache=${SPACK_CACHE_BUILD}/build_cache \ python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
--remote-cache-type=oci \ --local-cache=${SPACK_CACHE_BUILD}/build_cache \
--remote-cache=${OCI_CACHE_PREFIX}/build_cache \ --remote-cache-type=oci \
--yashchiki-home=${YASHCHIKI_HOME} \ --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
missing_paths_buildresults.dat ${CACHE_SPECFILE} --yashchiki-home=${YASHCHIKI_HOME} \
missing_paths_buildresults.dat ${CACHE_SPECFILE}
fi
spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && ( spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && (
# fetch all sources but delay exit code handling # fetch all sources but delay exit code handling
spack fetch --dependencies --missing && ret=$? || ret=$?; spack fetch --dependencies --missing && ret=$? || ret=$?;
# push freshly fetched sources to remote cache if [ -n "${OCI_CACHE_PREFIX}" ]; then
if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then # push freshly fetched sources to remote cache
echo "Performing update of the source cache" if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
python3 ${YASHCHIKI_HOME}/update_cached_sources.py \ echo "Performing update of the source cache"
--local-cache=${SPACK_CACHE_SOURCE} \ python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
--remote-cache-type=oci \ --local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \ --remote-cache-type=oci \
missing_paths_sources.dat; --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
else missing_paths_sources.dat;
echo "Updating of the source cache disabled." else
echo "Updating of the source cache disabled."
fi
fi fi
if [ $ret -ne 0 ]; then if [ $ret -ne 0 ]; then
(exit $ret) (exit $ret)
...@@ -125,15 +128,17 @@ spack-python -c "exit(not len(spack.environment.active_environment().uninstalled ...@@ -125,15 +128,17 @@ spack-python -c "exit(not len(spack.environment.active_environment().uninstalled
# (Note: spack expects `build_cache/` below the folder we specify here # (Note: spack expects `build_cache/` below the folder we specify here
spack mirror add local_cache ${SPACK_CACHE_BUILD} spack mirror add local_cache ${SPACK_CACHE_BUILD}
# record the state of installed/uninstalled packages before actually installing them if [ -n "${OCI_CACHE_PREFIX}" ]; then
dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE}) # record the state of installed/uninstalled packages before actually installing them
dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
fi
# install the environment, use 2 jobs to reduce the amount of required RAM # install the environment, use 2 jobs to reduce the amount of required RAM
# delay exit code until we have updated the cache below # delay exit code until we have updated the cache below
spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$? spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$?
# no need to update the local cache nor the remote cache if we don't want to update # no need to update the local cache nor the remote cache if we don't want to update
if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then if [ -n "${OCI_CACHE_PREFIX}" ] && [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
# push previously missing (but now installed) packages to the local cache # push previously missing (but now installed) packages to the local cache
for dag_hash in $dag_hashes_pre_install; do for dag_hash in $dag_hashes_pre_install; do
spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$? spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$?
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment