Skip to content
Snippets Groups Projects
Commit 48541cc2 authored by Eric Müller's avatar Eric Müller :mountain_bicyclist:
Browse files

fix: omit OCI cache handling code if cache isn't specified

Changes in CI file allow to use existing spack deployments w/o cache.
parent a000bba0
No related branches found
No related tags found
2 merge requests!651create new experimental release,!627Disable OCI cache usage for jobs w/o install prefix "/esd"
......@@ -16,7 +16,7 @@ EBRAINS_REPO=$3 # location of ebrains-spack-builds repository
EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated
UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional)
UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results
OCI_CACHE_PREFIX=$7
export OCI_CACHE_PREFIX=$7
# specify location of .spack dir (by default in ~)
# this is where cache and configuration settings are stored
......@@ -82,39 +82,42 @@ spack concretize --force --fresh --test root
export CACHE_SPECFILE=${CACHE_SPECFILE:-"env_specfile.yaml"}
export YASHCHIKI_HOME=${EBRAINS_REPO}/vendor/yashchiki
export OCI_CACHE_PREFIX=${OCI_CACHE_PREFIX:-${HARBOR_HOST:-"docker-registry.ebrains.eu"}/${HARBOR_PROJECT:-"esd"}/master}
export SPACK_CACHE_SOURCE=${SPACK_CACHE_SOURCE:-${INSTALLATION_ROOT}/spack/var/spack/cache}
export SPACK_CACHE_BUILD=${SPACK_CACHE_BUILD:-${INSTALLATION_ROOT}/spack/var/spack/cache}
# dump dag to file
spack spec -y > "${CACHE_SPECFILE}"
# fetch missing sources (if packages not yet installed)
python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
--local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \
--yashchiki-home=${YASHCHIKI_HOME} \
missing_paths_sources.dat ${CACHE_SPECFILE}
# fetch missing build results (if packages not yet installed)
python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
--local-cache=${SPACK_CACHE_BUILD}/build_cache \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/build_cache \
--yashchiki-home=${YASHCHIKI_HOME} \
missing_paths_buildresults.dat ${CACHE_SPECFILE}
if [ -n "${OCI_CACHE_PREFIX}" ]; then
# fetch missing sources (if packages not yet installed)
python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
--local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \
--yashchiki-home=${YASHCHIKI_HOME} \
missing_paths_sources.dat ${CACHE_SPECFILE}
# fetch missing build results (if packages not yet installed)
python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
--local-cache=${SPACK_CACHE_BUILD}/build_cache \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/build_cache \
--yashchiki-home=${YASHCHIKI_HOME} \
missing_paths_buildresults.dat ${CACHE_SPECFILE}
fi
spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && (
# fetch all sources but delay exit code handling
spack fetch --dependencies --missing && ret=$? || ret=$?;
# push freshly fetched sources to remote cache
if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
echo "Performing update of the source cache"
python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
--local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \
missing_paths_sources.dat;
else
echo "Updating of the source cache disabled."
if [ -n "${OCI_CACHE_PREFIX}" ]; then
# push freshly fetched sources to remote cache
if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
echo "Performing update of the source cache"
python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
--local-cache=${SPACK_CACHE_SOURCE} \
--remote-cache-type=oci \
--remote-cache=${OCI_CACHE_PREFIX}/source_cache \
missing_paths_sources.dat;
else
echo "Updating of the source cache disabled."
fi
fi
if [ $ret -ne 0 ]; then
(exit $ret)
......@@ -125,15 +128,17 @@ spack-python -c "exit(not len(spack.environment.active_environment().uninstalled
# (Note: spack expects `build_cache/` below the folder we specify here
spack mirror add local_cache ${SPACK_CACHE_BUILD}
# record the state of installed/uninstalled packages before actually installing them
dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
if [ -n "${OCI_CACHE_PREFIX}" ]; then
# record the state of installed/uninstalled packages before actually installing them
dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
fi
# install the environment, use 2 jobs to reduce the amount of required RAM
# delay exit code until we have updated the cache below
spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$?
# no need to update the local cache nor the remote cache if we don't want to update
if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
if [ -n "${OCI_CACHE_PREFIX}" ] && [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
# push previously missing (but now installed) packages to the local cache
for dag_hash in $dag_hashes_pre_install; do
spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$?
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment