# This is a combination of 2 commits.

# This is the 1st commit message:

feat(CI,JUSUF): transparent base image

# The commit message #2 will be skipped:

# fixu

.gitlab-ci.yml

@@ -272,10 +272,13 @@ build-esd-image_JUSUF:
   variables:
     CI_SPACK_ENV: esd
     SPACK_JOBS: 4
-    # we access the branch-specific cache here (and also update it!)
-    OCI_CACHE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}_jusuf
+    # no OCI caching for JUSUF builds, we rely on shared fs
+    OCI_CACHE_PREFIX: ""
     UPDATE_SPACK_OCI_CACHES: false
-    OCI_IMAGE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}_jusuf/image
+    OCI_IMAGE_PREFIX: ""
+    # for now, we directly push to a local buildcache
+    UPDATE_SPACK_LOCAL_CACHE: true
+    FINALFLUSH_SPACK_LOCAL_CACHE: false
     INSTALLATION_ROOT: /esd
     SANDBOX_ROOT_RELATIVE: esd_image
     TMPDIR: /dev/shm/ecm_gitlab_ci

install_spack_env.sh

@@ -39,6 +39,10 @@ if [ ! -d ${INSTALLATION_ROOT} ]; then
     mkdir -p ${INSTALLATION_ROOT}
 fi
 
+if [ $SYSTEMNAME == jusuf ]; then
+    module load Python
+fi
+
 # initial setup: use spack submodule if spack dir doesn't already exist
 SPACK_ROOT_EXISTED=1
 if [ ! -d ${CI_SPACK_ROOT} ]; then
@@ -75,6 +79,12 @@ then
   spack repo add $EBRAINS_REPO
 fi
 
+if [ $SYSTEMNAME == jusuf ]; then
+    spack bootstrap status || true
+    spack --insecure bootstrap now
+    spack config add config:verify_ssl:false
+fi
+
 # make sure all fetching/clingo stuff happens before anything else
 spack spec aida
 
@@ -185,12 +195,13 @@ spack-python -c "exit(not len(spack.environment.active_environment().uninstalled
     else
         echo "Updating of the source cache disabled."
     fi
+    # FIXME: local source cache update?
     if [ $ret -ne 0 ]; then
         (exit $ret)
     fi
 )
 
-if [ -n "${OCI_CACHE_PREFIX}" ]; then
+if [ -n "${UPDATE_SPACK_LOCAL_CACHES}" ]; then
     # record the state of installed/uninstalled packages before actually installing them
     dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
 fi
@@ -199,8 +210,8 @@ fi
 # delay exit code until we have updated the cache below
 spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$?
 
-# no need to update the local cache nor the remote cache if we don't want to update
-if [ -n "${OCI_CACHE_PREFIX}" ] && [ "${UPDATE_SPACK_OCI_CACHES:-false}" = "true" ]; then
+# only if we update the local cache we will look at the remote cache
+if [ -n "${UPDATE_SPACK_LOCAL_CACHE}" ]; then
     # push previously missing (but now installed) packages to the local cache
     for dag_hash in $dag_hashes_pre_install; do
         spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$?
@@ -210,15 +221,19 @@ if [ -n "${OCI_CACHE_PREFIX}" ] && [ "${UPDATE_SPACK_OCI_CACHES:-false}" = "true
         fi
     done
 
-    # upload packages from local to remote cache
-    echo "Performing update of the build cache"
-    python3 ${YASHCHIKI_HOME}/update_cached_buildresults.py \
-        --local-cache=${SPACK_CACHE_BUILD}/build_cache \
-        --remote-cache-type=oci \
-        --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
-        /tmp/missing_paths_buildresults.dat
+    if [ -n "${OCI_CACHE_PREFIX}" ] && [ "${UPDATE_SPACK_OCI_CACHES:-false}" = "true" ]; then
+        # upload packages from local to remote cache
+        echo "Performing update of the build cache"
+        python3 ${YASHCHIKI_HOME}/update_cached_buildresults.py \
+            --local-cache=${SPACK_CACHE_BUILD}/build_cache \
+            --remote-cache-type=oci \
+            --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
+            /tmp/missing_paths_buildresults.dat
+    else
+        echo "Updating of remote build cache disabled."
+    fi
 else
-    echo "Updating of the build cache disabled."
+    echo "Updating of the local build cache disabled."
 fi
 
 # propagate spack install exit code
@@ -227,7 +242,7 @@ if [ $spack_install_ret -ne 0 ]; then
 fi
 
 # remove local cache content
-if [ -d ${SPACK_CACHE_BUILD} ]; then
+if [ "${FINALFLUSH_SPACK_LOCAL_CACHE:-false}" = "true" ] && [ -d ${SPACK_CACHE_BUILD} ]; then
     spack mirror destroy --mirror-name local_cache
 fi