diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59cc11f9c4bcc131cf281a246756d7b5acf011c2..e76022477214ebe2be62e6e3e5207e4385810b51 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ variables:
# start a k8s Job that will build the Spack environment
.deploy-build-environment:
variables:
- OCI_CACHE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}
+ OCI_CACHE_PREFIX: ""
UPDATE_SPACK_OCI_CACHES: false
stage: build
tags:
@@ -186,7 +186,7 @@ build-spack-env-on-runner:
variables:
SPACK_DEV_ENV: ebrains-dev
SPACK_JOBS: 2
- OCI_CACHE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/master
+ OCI_CACHE_PREFIX: ""
UPDATE_SPACK_OCI_CACHES: false
script:
# deactivate environment views (we don't need them for the test build-job)
@@ -221,15 +221,15 @@ sync-gitlab-spack-instance:
variables:
SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds
SPACK_JOBS: 4
- OCI_CACHE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}
- UPDATE_SPACK_OCI_CACHES: true
+ OCI_CACHE_PREFIX: ""
+ UPDATE_SPACK_OCI_CACHES: false
script:
- SPACK_NFS_ENV=${CI_COMMIT_BRANCH//./-}
# create spack dir if it doesn't exist
- mkdir -p $SPACK_PATH_GITLAB
# get latest state of EBRAINS repo
- rm -rf $SPACK_REPO_PATH && cp -r $CI_PROJECT_DIR $SPACK_REPO_PATH
- # run installation script and set UPDATE to true
+ # run installation script
- bash install_spack_env.sh $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_REPO_PATH $SPACK_NFS_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
# create kernel spec, so that the environment can be used in gitlab CI jobs
- RELEASE_NAME=$(case $CI_COMMIT_BRANCH in experimental_rel) echo ebrains-experimental;; ebrains*) echo ${CI_COMMIT_BRANCH:0:10}.${CI_COMMIT_BRANCH:11};; *) echo $CI_COMMIT_BRANCH;; esac);
@@ -247,7 +247,7 @@ sync-gitlab-spack-instance:
- spack_logs
when: always
rules:
- # branches that update the gitlab-runner upstream (read-only) installation and the spack OCI caches
+ # branches that update the gitlab-runner upstream (read-only) installation
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "experimental_rel" || $CI_COMMIT_BRANCH =~ /^ebrains/) && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule"
when: manual
diff --git a/install_spack_env.sh b/install_spack_env.sh
index c21db599277cc9ae4975da8071c50a42c1268549..0784e35b370c6380343a5bf34878c99a2c67e6fb 100644
--- a/install_spack_env.sh
+++ b/install_spack_env.sh
@@ -16,7 +16,7 @@ EBRAINS_REPO=$3 # location of ebrains-spack-builds repository
EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated
UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional)
UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results
-OCI_CACHE_PREFIX=$7
+export OCI_CACHE_PREFIX=$7
# specify location of .spack dir (by default in ~)
# this is where cache and configuration settings are stored
@@ -82,39 +82,42 @@ spack concretize --force --fresh --test root
export CACHE_SPECFILE=${CACHE_SPECFILE:-"env_specfile.yaml"}
export YASHCHIKI_HOME=${EBRAINS_REPO}/vendor/yashchiki
-export OCI_CACHE_PREFIX=${OCI_CACHE_PREFIX:-${HARBOR_HOST:-"docker-registry.ebrains.eu"}/${HARBOR_PROJECT:-"esd"}/master}
export SPACK_CACHE_SOURCE=${SPACK_CACHE_SOURCE:-${INSTALLATION_ROOT}/spack/var/spack/cache}
export SPACK_CACHE_BUILD=${SPACK_CACHE_BUILD:-${INSTALLATION_ROOT}/spack/var/spack/cache}
# dump dag to file
spack spec -y > "${CACHE_SPECFILE}"
-# fetch missing sources (if packages not yet installed)
-python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
- --local-cache=${SPACK_CACHE_SOURCE} \
- --remote-cache-type=oci \
- --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
- --yashchiki-home=${YASHCHIKI_HOME} \
- missing_paths_sources.dat ${CACHE_SPECFILE}
-# fetch missing build results (if packages not yet installed)
-python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
- --local-cache=${SPACK_CACHE_BUILD}/build_cache \
- --remote-cache-type=oci \
- --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
- --yashchiki-home=${YASHCHIKI_HOME} \
- missing_paths_buildresults.dat ${CACHE_SPECFILE}
+if [ -n "${OCI_CACHE_PREFIX}" ]; then
+ # fetch missing sources (if packages not yet installed)
+ python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
+ --local-cache=${SPACK_CACHE_SOURCE} \
+ --remote-cache-type=oci \
+ --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
+ --yashchiki-home=${YASHCHIKI_HOME} \
+ missing_paths_sources.dat ${CACHE_SPECFILE}
+ # fetch missing build results (if packages not yet installed)
+ python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
+ --local-cache=${SPACK_CACHE_BUILD}/build_cache \
+ --remote-cache-type=oci \
+ --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
+ --yashchiki-home=${YASHCHIKI_HOME} \
+ missing_paths_buildresults.dat ${CACHE_SPECFILE}
+fi
spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && (
# fetch all sources but delay exit code handling
spack fetch --dependencies --missing && ret=$? || ret=$?;
- # push freshly fetched sources to remote cache
- if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
- echo "Performing update of the source cache"
- python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
- --local-cache=${SPACK_CACHE_SOURCE} \
- --remote-cache-type=oci \
- --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
- missing_paths_sources.dat;
- else
- echo "Updating of the source cache disabled."
+ if [ -n "${OCI_CACHE_PREFIX}" ]; then
+ # push freshly fetched sources to remote cache
+ if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
+ echo "Performing update of the source cache"
+ python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
+ --local-cache=${SPACK_CACHE_SOURCE} \
+ --remote-cache-type=oci \
+ --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
+ missing_paths_sources.dat;
+ else
+ echo "Updating of the source cache disabled."
+ fi
fi
if [ $ret -ne 0 ]; then
(exit $ret)
@@ -125,15 +128,17 @@ spack-python -c "exit(not len(spack.environment.active_environment().uninstalled
# (Note: spack expects `build_cache/` below the folder we specify here
spack mirror add local_cache ${SPACK_CACHE_BUILD}
-# record the state of installed/uninstalled packages before actually installing them
-dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
+if [ -n "${OCI_CACHE_PREFIX}" ]; then
+ # record the state of installed/uninstalled packages before actually installing them
+ dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
+fi
# install the environment, use 2 jobs to reduce the amount of required RAM
# delay exit code until we have updated the cache below
spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$?
# no need to update the local cache nor the remote cache if we don't want to update
-if [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
+if [ -n "${OCI_CACHE_PREFIX}" ] && [ ${UPDATE_SPACK_OCI_CACHES:-false} = "true" ]; then
# push previously missing (but now installed) packages to the local cache
for dag_hash in $dag_hashes_pre_install; do
spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$?