.gitlab/merge_request_templates/new-package.md

+### Summary
+
+|               |                                        |
+|---------------|----------------------------------------|
+| Package       | <!-- Spack package name -->            |
+| Contact point | <!-- maintainer GitLab handle -->      |
+| KG entry      | <!-- insert link -->                   |
+
+
+### Checks
+
+<!-- please download the EBRAINS Software Quality checklist, fill it and replace the file below -->
+* [ ] Software Quality Checklist: [SQ-Checklist.pdf](https://drive.ebrains.eu/d/6061531326d048308823/files/?p=%2FSQ-Checklist.pdf) (level: <!-- passing, silver, gold -->)
+* [ ] Current maintainer is listed first in `package.py`
+* [ ] No pinned dependency versions
+* [ ] Post-installation tests are defined
\ No newline at end of file

.gitlab/merge_request_templates/new-version.md

+### Summary
+
+|               |                                        |
+|---------------|----------------------------------------|
+| Package       | <!-- Spack package name -->            |
+| Version added | <!-- version number -->                |
+| Contact point | <!-- maintainer GitLab handle -->      |
+| KG entry      | <!-- insert link -->                   |
+
+
+### Checks
+
+<!-- please download the EBRAINS Software Quality checklist, fill it and replace the file below -->
+* [ ] Software Quality Checklist: [SQ-Checklist.pdf](https://drive.ebrains.eu/d/6061531326d048308823/files/?p=%2FSQ-Checklist.pdf) (level: <!-- passing, silver, gold -->)
+* [ ] Current maintainer is listed first in `package.py`
+* [ ] No pinned dependency versions
+* [ ] Post-installation tests are defined
\ No newline at end of file

.gitmodules

+[submodule "vendor/spack"]
+	path = vendor/spack
+	url = https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack.git
+	shallow = true
+[submodule "vendor/yashchiki"]
+	path = vendor/yashchiki
+	url = https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/dedal
+	shallow = true

README.md

@@ -38,9 +38,9 @@ Clone this repository. You can use the `ebrains-yy-mm` branches to install the E
 git clone --branch {branch-name} https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git
 ```
 
-Clone Spack. We currently use version v0.21.1:
+Clone Spack. We currently use version v0.23.1:
 ```
-git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch v0.20.0 https://github.com/spack/spack
+git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch v0.23.1 https://github.com/spack/spack
 ```
 
 Activate Spack:
@@ -53,11 +53,6 @@ Add the project repository to your Spack environment:
 spack repo add ebrains-spack-builds
 ```
 
-Create the environment:
-```
-spack env create -d ebrains-spack-builds/
-```
-
 Define your site-specific configurations:
 ```
 export SYSTEMNAME=<your-system-name>

create_JupyterLab_kernel.sh

@@ -7,6 +7,8 @@
 #                 loaded by all users.
 # ===========================================================================================================
 
+set -euo pipefail
+
 INSTALLATION_ROOT=$1
 EBRAINS_SPACK_ENV=$2
 RELEASE_NAME=$3
@@ -35,12 +37,17 @@ cp $INSTALLATION_ROOT/spack/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.s
 # and the location of python modules installed in the base docker Collab image
 cat <<EOF >> $KERNEL_PATH/bin/env.sh
 export PATH=\$PATH:/opt/app-root/src/.local/bin
-export PYTHONPATH=\$PYTHONPATH:/opt/app-root/src/.local/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages
+export PYTHONPATH=\$PYTHONPATH:/opt/conda/lib/python3.11/site-packages
 export R_LIBS_USER=/opt/app-root/src/.local/lib/R/site-library
 mkdir -p \$R_LIBS_USER
 export R_LIBS=\$R_LIBS_USER:\$R_LIBS
 EOF
 
+# ... and set the SYSTEMNAME env var, to use Spack commands inside the kernel
+cat <<EOF >> $KERNEL_PATH/bin/env.sh
+export SYSTEMNAME=ebrainslab
+EOF
+
 # (4) create startup script for python kernel
 cat $KERNEL_PATH/bin/env.sh > $KERNEL_PATH/bin/env_python.sh
 echo "python -m ipykernel_launcher -f \$@" >> $KERNEL_PATH/bin/env_python.sh

create_job.sh

@@ -2,20 +2,20 @@
 
 # ===========================================================================================================
 # title         : create_job.sh
-# usage         : ./create_job.sh $OC_JOB_ID $BUILD_ENV_DOCKER_IMAGE $OP $INSTALLATION_ROOT $SPACK_VERSION
-#                 $SPACK_ENV $BRANCH $RELEASE_NAME $LAB_KERNEL_ROOT
+# usage         : ./create_job.sh $OC_JOB_ID $BUILD_ENV_DOCKER_IMAGE $INSTALLATION_ROOT
+#                 $SPACK_ENV $COMMIT_SHA $RELEASE_NAME $LAB_KERNEL_ROOT
 # description   : creates OKD job yaml file that builds/updates spack environment and creates Lab kernel
 # ===========================================================================================================
 
 OC_JOB_ID=$1
 BUILD_ENV_DOCKER_IMAGE=$2
-OP=$3
-INSTALLATION_ROOT=$4
-SPACK_VERSION=$5
-SPACK_ENV=$6
-BRANCH=$7
-RELEASE_NAME=$8
-LAB_KERNEL_ROOT=$9
+INSTALLATION_ROOT=$3
+EBRAINS_SPACK_ENV=$4
+COMMIT_SHA=$5
+RELEASE_NAME=$6
+LAB_KERNEL_ROOT=$7
+UPDATE_SPACK_OCI_CACHES=$8
+OCI_CACHE_PREFIX=$9
 
 cat <<EOT >> simplejob.yml
 apiVersion: batch/v1
@@ -34,26 +34,77 @@ spec:
       - name: simplejob
         image: ${BUILD_ENV_DOCKER_IMAGE}
         imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: '8'
+            memory: '32Gi'
+          requests:
+            cpu: '4'
+            memory: '20Gi'
         volumeMounts:
           - name: sharedbin
             mountPath: /srv
-        command: ["/usr/local/bin/deploy-build-env.sh", "$OP", "$INSTALLATION_ROOT", "$SPACK_VERSION", "$SPACK_ENV", "$BRANCH", "$RELEASE_NAME", "$LAB_KERNEL_ROOT"]
+          - name: tmp
+            mountPath: /tmp
+        command:
+        - /bin/bash
+        - -c
+        - |
+          # create root dir if it doesn't exist
+          mkdir -p \$INSTALLATION_ROOT
+          
+          # reset spack repository dir by cloning the selected version
+          if [ ! -d \$EBRAINS_REPO_PATH ]; then git clone ${CI_PROJECT_URL} --recurse-submodules \$EBRAINS_REPO_PATH; fi
+          cd \$EBRAINS_REPO_PATH
+          git fetch origin
+          git reset --hard \$COMMIT_SHA
+          git submodule update --force
+
+          # reset build error log dir (delete previous logs to save space)
+          rm -rf \$BUILD_LOGS_DIR
+          mkdir -p \$BUILD_LOGS_DIR
+          cd \$BUILD_LOGS_DIR
+          
+          # run installation script
+          bash \$EBRAINS_REPO_PATH/install_spack_env.sh \$SPACK_JOBS \$INSTALLATION_ROOT \$EBRAINS_REPO_PATH \$EBRAINS_SPACK_ENV "" \$UPDATE_SPACK_OCI_CACHES \$OCI_CACHE_PREFIX
+          
+          if [ \$? -eq 0 ]
+          then
+            # build process succeeded - create or update kernel on the NFS based on the current spack environment
+            bash \$EBRAINS_REPO_PATH/create_JupyterLab_kernel.sh \$INSTALLATION_ROOT \$EBRAINS_SPACK_ENV \$RELEASE_NAME \$LAB_KERNEL_ROOT && exit 0
+          else
+            # build process failed - keep spack build logs and fail the pipeline
+            cp -r /tmp/spack/spack-stage/* \$BUILD_LOGS_DIR
+            exit 1
+          fi
         env:
           - name: SYSTEMNAME
             value: ebrainslab
-          - name: GITLAB_USER
-            valueFrom:
-              secretKeyRef:
-                name: spack-repo-gitlab-token
-                key: username
-          - name: GITLAB_TOKEN
-            valueFrom:
-              secretKeyRef:
-                name: spack-repo-gitlab-token
-                key: password
+          - name: INSTALLATION_ROOT
+            value: $INSTALLATION_ROOT
+          - name: EBRAINS_SPACK_ENV
+            value: $EBRAINS_SPACK_ENV
+          - name: UPDATE_SPACK_OCI_CACHES
+            value: '$UPDATE_SPACK_OCI_CACHES'
+          - name: OCI_CACHE_PREFIX
+            value: $OCI_CACHE_PREFIX
+          - name: COMMIT_SHA
+            value: $COMMIT_SHA
+          - name: RELEASE_NAME
+            value: $RELEASE_NAME
+          - name: LAB_KERNEL_ROOT
+            value: $LAB_KERNEL_ROOT
+          - name: BUILD_LOGS_DIR
+            value: /srv/build_logs/$EBRAINS_SPACK_ENV
+          - name: EBRAINS_REPO_PATH
+            value: $INSTALLATION_ROOT/ebrains-spack-builds
+          - name: SPACK_JOBS
+            value: '6'
       volumes:
         - name: sharedbin
           persistentVolumeClaim:
             claimName: shared-binaries
+        - name: tmp
+          emptyDir: {}
       restartPolicy: Never
 EOT

create_job_widget_script.sh

+#!/bin/bash
+
+# ===========================================================================================================
+# title         : create_job_widget_script.sh
+# usage         : ./create_job_widget_script.sh $OC_JOB_ID $RUN_ENV_DOCKER_IMAGE $INSTALLATION_ROOT
+#                 $SPACK_ENV $RELEASE_NAME $LAB_KERNEL_ROOT
+# description   : creates job file that generates a script to load the jupyter extensions for a given env
+# ===========================================================================================================
+
+OC_JOB_ID=$1
+RUN_ENV_DOCKER_IMAGE=$2
+INSTALLATION_ROOT=$3
+EBRAINS_SPACK_ENV=$4
+RELEASE_NAME=$5
+LAB_KERNEL_ROOT=$6
+
+cat <<EOT >> widget-script.yml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: widget-script${OC_JOB_ID}
+spec:
+  parallelism: 1
+  completions: 1
+  backoffLimit: 0
+  template:
+    spec:
+      containers:
+      - name: widget-script
+        image: ${RUN_ENV_DOCKER_IMAGE}
+        imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: '1'
+            memory: '1Gi'
+          requests:
+            cpu: '0.5'
+            memory: '500Mi'
+        volumeMounts:
+          - name: sharedbin
+            mountPath: /srv
+        command:
+        - /bin/bash
+        - -c
+        - |
+          . \$INSTALLATION_ROOT/spack/share/spack/setup-env.sh
+          spack env activate --without-view \$EBRAINS_SPACK_ENV
+          KERNEL_PATH=\$LAB_KERNEL_ROOT/\$(echo "\$RELEASE_NAME" | tr '[:upper:]' '[:lower:]')
+          spack load --sh --first clb-nb-utils py-pip py-tvb-ext-bucket py-tvb-ext-unicore py-tvb-ext-xircuits > \$KERNEL_PATH/bin/widget_activation.sh
+        env:
+          - name: SYSTEMNAME
+            value: ebrainslab
+          - name: SPACK_DISABLE_LOCAL_CONFIG
+            value: "true"
+          - name: INSTALLATION_ROOT
+            value: "$INSTALLATION_ROOT"
+          - name: EBRAINS_SPACK_ENV
+            value: "$EBRAINS_SPACK_ENV"
+          - name: RELEASE_NAME
+            value: "$RELEASE_NAME"
+          - name: LAB_KERNEL_ROOT
+            value: "$LAB_KERNEL_ROOT"
+      volumes:
+        - name: sharedbin
+          persistentVolumeClaim:
+            claimName: shared-binaries
+      restartPolicy: Never
+EOT

install_spack_env.sh

 #!/bin/bash
 
-# ===============================================================================================================================
+# =========================================================================================================================================
 # title         : install_spack_env.sh
-# usage         : ./install_spack_env.sh $INSTALLATION_ROOT $SPACK_VERSION $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE
+# usage         : ./install_spack_env.sh $SPACK_JOBS $INSTALLATION_ROOT $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE \
+#                                        $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
 # description   : installs or updates the spack environment defined in the EBRAINS spack repo
 #                 (if the specified spack instance doesn't exist, it also creates it)
-# ===============================================================================================================================
+# =========================================================================================================================================
 
-INSTALLATION_ROOT=$1  # where to set up the installation
-SPACK_VERSION=$2      # which spack version to use
+set -eo pipefail
+
+SPACK_JOBS=$1         # number of jobs
+INSTALLATION_ROOT=$2  # where to set up the installation
 EBRAINS_REPO=$3       # location of ebrains-spack-builds repository
 EBRAINS_SPACK_ENV=$4  # name of EBRAINS Spack environment to be created/updated
 UPSTREAM_INSTANCE=$5  # path to Spack instance to use as upstream (optional)
+UPDATE_SPACK_OCI_CACHES=$6 # "true" enables updating the OCI cache for spack sources and build results
+export OCI_CACHE_PREFIX=$7
 
-SPACK_REPO=https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack.git
-SPACK_VERSION_EBRAINS=${SPACK_VERSION}_ebrains24.04
+# make sure spack uses the symlinked folder as path
+export CI_SPACK_ROOT=${INSTALLATION_ROOT}/spack
 
-# specify location of .spack dir (by default in ~)
-# this is where cache and configuration settings are stored
-export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
-export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
+# disable local configuration and cache directories
+export SPACK_DISABLE_LOCAL_CONFIG=true
+export SPACK_USER_CACHE_PATH=/tmp/spack
 
 # define SYSTEMNAME variable in sites where it's not already defined
 export SYSTEMNAME=${SYSTEMNAME:-${HPC_SYSTEM:-$BSC_MACHINE}}
 
-# initial setup: clone spack if spack dir doesn't already exist
-if [ ! -d $INSTALLATION_ROOT/spack ]
-then
-  git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION_EBRAINS $SPACK_REPO $INSTALLATION_ROOT/spack
-  # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
-  # quick fix for that: (TODO: investigate more and open PR)
-  sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $INSTALLATION_ROOT/spack/lib/spack/llnl/util/tty/log.py
+# cache related variables
+export CACHE_SPECFILE=${CACHE_SPECFILE:-"env_specfile.yaml"}
+export YASHCHIKI_HOME=${EBRAINS_REPO}/vendor/yashchiki
+export SPACK_CACHE_SOURCE=${SPACK_CACHE_SOURCE:-${CI_SPACK_ROOT}/var/spack/cache}
+export SPACK_CACHE_BUILD=${SPACK_CACHE_BUILD:-${CI_SPACK_ROOT}/var/spack/cache}
+
+if [ ! -d ${INSTALLATION_ROOT} ]; then
+    mkdir -p ${INSTALLATION_ROOT}
 fi
 
-if [[ $UPSTREAM_INSTANCE ]]
-then
-  cat <<EOF > $INSTALLATION_ROOT/spack/etc/spack/defaults/upstreams.yaml
-upstreams:
-  upstream-spack-instance:
-    install_tree: $UPSTREAM_INSTANCE/spack/opt/spack
-EOF
+# initial setup: use spack submodule if spack dir doesn't already exist
+SPACK_ROOT_EXISTED=1
+if [ ! -d ${CI_SPACK_ROOT} ]; then
+  ln -s ${EBRAINS_REPO}/vendor/spack ${CI_SPACK_ROOT}
+  SPACK_ROOT_EXISTED=0
 fi
 
 # activate Spack
-source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
+source ${CI_SPACK_ROOT}/share/spack/setup-env.sh
+
+if [[ $UPSTREAM_INSTANCE ]]; then
+    UPSTREAM_PREFIX=$(find $UPSTREAM_INSTANCE/spack/opt/spack/ -type d -name ".spack-db" 2>/dev/null | xargs -I {} dirname {})
+    spack config add upstreams:upstream-spack-instance:install_tree:$UPSTREAM_PREFIX
+fi
+
+if [ "${SPACK_ROOT_EXISTED}" -eq 0 ]; then
+    # for caching purposes it's nice if we can relocate into long paths, but we
+    # can't do that for existing installations -> else path
+    # ECM (2025-01-23) true seems to yield too large paths for some packages (e.g., gcc)
+    spack config add config:install_tree:padded_length:128
+fi
 
 # add repo if it does not exist
 if [[ ! $(spack repo list | grep ebrains-spack-builds$) ]]
@@ -51,37 +66,119 @@ then
   spack repo add $EBRAINS_REPO
 fi
 
-# install platform compiler (extract version from packages.yaml)
-if [ $SYSTEMNAME == ebrainslab ]
-then
-  EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }')
-  spack compiler find
-  spack load $EBRAINS_SPACK_COMPILER || { spack install $EBRAINS_SPACK_COMPILER; spack load $EBRAINS_SPACK_COMPILER; }
+# make sure all fetching/clingo stuff happens before anything else
+spack spec aida
+
+# rebuild spack's database (could be an debugging session)
+spack reindex
+
+# add local mirror if it does not exist
+if [[ ! $(spack mirror list | grep local_cache) ]]; then
+    # TODO for newer spack versions, add: --autopush --unsigned, drop create cache command below
+    # (Note: spack expects `build_cache/` below the folder we specify here
+    spack mirror add local_cache ${SPACK_CACHE_BUILD}
 fi
 
 spack compiler find
 
 # create environment if it does not exist
-if [ ! -d "$SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV" ]
+if [ ! -d "${CI_SPACK_ROOT}/var/spack/environments/$EBRAINS_SPACK_ENV" ]
 then
   spack env create $EBRAINS_SPACK_ENV
 fi
 
 # update environment site-configs
-rm -rf $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV
+rm -rf ${CI_SPACK_ROOT}/var/spack/environments/$EBRAINS_SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config ${CI_SPACK_ROOT}/var/spack/environments/$EBRAINS_SPACK_ENV
 # update spack.yaml: merge top-level and site-specific spack.yaml files
 spack-python $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
-cp /tmp/spack.yaml $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/
+cp /tmp/spack.yaml ${CI_SPACK_ROOT}/var/spack/environments/$EBRAINS_SPACK_ENV/
 
 # activate environment
 spack env activate --without-view $EBRAINS_SPACK_ENV
 
-# fetch all sources
 spack concretize --force --fresh --test root
-spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && spack fetch --dependencies --missing
+
+# dump dag to file
+spack spec -y > "${CACHE_SPECFILE}"
+if [ -n "${OCI_CACHE_PREFIX}" ]; then
+    # fetch missing sources (if packages not yet installed)
+    python3 ${YASHCHIKI_HOME}/fetch_cached_sources.py \
+        --local-cache=${SPACK_CACHE_SOURCE} \
+        --remote-cache-type=oci \
+        --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
+        --yashchiki-home=${YASHCHIKI_HOME} \
+        /tmp/missing_paths_sources.dat ${CACHE_SPECFILE}
+    # fetch missing build results (if packages not yet installed)
+    python3 ${YASHCHIKI_HOME}/fetch_cached_buildresults.py \
+        --local-cache=${SPACK_CACHE_BUILD}/build_cache \
+        --remote-cache-type=oci \
+        --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
+        --yashchiki-home=${YASHCHIKI_HOME} \
+        /tmp/missing_paths_buildresults.dat ${CACHE_SPECFILE}
+fi
+spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && (
+    # fetch all sources but delay exit code handling
+    spack fetch --dependencies --missing && ret=$? || ret=$?;
+    if [ -n "${OCI_CACHE_PREFIX}" ] && [ "${UPDATE_SPACK_OCI_CACHES:-false}" = "true" ]; then
+        # push freshly fetched sources to remote cache
+        echo "Performing update of the source cache"
+        python3 ${YASHCHIKI_HOME}/update_cached_sources.py \
+            --local-cache=${SPACK_CACHE_SOURCE} \
+            --remote-cache-type=oci \
+            --remote-cache=${OCI_CACHE_PREFIX}/source_cache \
+            /tmp/missing_paths_sources.dat;
+    else
+        echo "Updating of the source cache disabled."
+    fi
+    # if [ "$ret" -ne 0 ]; then
+    #     (exit $ret)
+    # fi
+)
+
+if [ -n "${OCI_CACHE_PREFIX}" ]; then
+    # record the state of installed/uninstalled packages before actually installing them
+    dag_hashes_pre_install=$(spack-python ${YASHCHIKI_HOME}/specfile_dag_hash.py ${CACHE_SPECFILE})
+fi
 
 # install the environment, use 2 jobs to reduce the amount of required RAM
-spack install -y -j2 --fresh --test root
+# delay exit code until we have updated the cache below
+spack install --no-check-signature -y -j$SPACK_JOBS --fresh --test root && spack_install_ret=$? || spack_install_ret=$?
+
+# no need to update the local cache nor the remote cache if we don't want to update
+if [ -n "${OCI_CACHE_PREFIX}" ] && [ "${UPDATE_SPACK_OCI_CACHES:-false}" = "true" ]; then
+    # push previously missing (but now installed) packages to the local cache
+    for dag_hash in $dag_hashes_pre_install; do
+        spack buildcache create --unsigned --only package ${SPACK_CACHE_BUILD} /${dag_hash} && ret=$? || ret=$?
+        if [ $ret -ne 0 ]; then
+            echo "Failed to push ${dag_hash}, trying to call spack find on it:"
+            spack find -Lvp /${dag_hash} || true
+        fi
+    done
+
+    # upload packages from local to remote cache
+    echo "Performing update of the build cache"
+    python3 ${YASHCHIKI_HOME}/update_cached_buildresults.py \
+        --local-cache=${SPACK_CACHE_BUILD}/build_cache \
+        --remote-cache-type=oci \
+        --remote-cache=${OCI_CACHE_PREFIX}/build_cache \
+        /tmp/missing_paths_buildresults.dat
+else
+    echo "Updating of the build cache disabled."
+fi
+
+# propagate spack install exit code
+if [ $spack_install_ret -ne 0 ]; then
+    (exit "$spack_install_ret")
+fi
+
+# remove local cache content
+if [ -d ${SPACK_CACHE_BUILD} ]; then
+    spack mirror destroy --mirror-name local_cache
+fi
+
+# TODO: when using spack remote OCI build caches require an index file
+#spack mirror add ebrains oci://docker-registry.ebrains.eu/esd/build_cache
+#spack buildcache list -a ebrains && ret=$? || ret=$?
 
 # rebuild spack's database
 spack reindex
@@ -90,7 +187,7 @@ spack reindex
 # this needs deactivating the environment first:
 spack env deactivate
 unset SPACK_LD_LIBRARY_PATH
-spack env activate --sh $EBRAINS_SPACK_ENV > $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh
+spack env activate --sh $EBRAINS_SPACK_ENV > ${CI_SPACK_ROOT}/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh
 
 # create modules files with spack
 # spack module tcl refresh -y

packages/acpype/package.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 class Acpype(PythonPackage):
     """A tool based in Python to use Antechamber to generate topologies for chemical 

packages/ambertools/package.py

@@ -3,52 +3,58 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
-class Ambertools (CMakePackage):
-    """AmberTools is a free, useful standalone package and a prerequisite for installing Amber itself.
-    The AmberTools suite is free of charge, and its components are mostly released under the GNU General Public License (GPL).
-    A few components are included that are in the public domain or which have other, open-source, licenses.
-    The libsander and libpbsa libraries use the LGPL license."""
 
-    # Set the homepage and download url
-    homepage = "http://ambermd.org/AmberTools.php"
-    url      = "http://ambermd.org/downloads/AmberTools22jlmrcc.tar.bz2"
+class Ambertools(CMakePackage):
+    """AmberTools is a free, useful standalone package and a prerequisite
+    for installing Amber itself. The AmberTools suite is free of charge,
+    and its components are mostly released under the GNU General Public
+    License (GPL). A few components are included that are in the public
+    domain or which have other, open-source, licenses. The libsander and
+    libpbsa libraries use the LGPL license."""
 
-    # Set the gitlab accounts of this package maintainers
-    maintainers = ['dbeltran', 'elmath']
+    homepage = "https://ambermd.org/AmberTools.php"
+    url = "https://ambermd.org/downloads/AmberTools22jlmrcc.tar.bz2"
 
-    version('22jlmrcc', sha256='1571d4e0f7d45b2a71dce5999fa875aea8c90ee219eb218d7916bf30ea229121')
+    maintainers("d-beltran")
 
-    # Dependencies
-    depends_on("flex",                      type="build") # This is necessary for sure (experimentally tested)
-    depends_on("bison",                     type="build") # This is necessary for sure (experimentally tested)
-    depends_on("tcsh",                      type="build")
-    depends_on("zlib",                      type=("build", "link", "run"))
-    depends_on("bzip2",                     type=("build", "run"))
-    depends_on("blas",                      type=("build", "run"))
-    depends_on("lapack",                    type=("build", "run"))
-    depends_on("arpack-ng",                 type=("build", "run"))
-    depends_on("netcdf-c",                  type=("build", "run"))
-    depends_on("netcdf-fortran",            type=("build", "run"))
-    depends_on("fftw",                      type=("build", "run"))
-    depends_on("readline",                  type=("build", "run"))
-    depends_on("netlib-xblas~plain_blas",   type=("build", "run"))
-    # specific variants needed for boost - from the build log "Could NOT find Boost (missing: thread system program_options iostreams regex timer chrono filesystem graph)"
-    depends_on("boost+thread+system+program_options+iostreams+regex+timer+chrono+filesystem+graph", type=("build", "run"))
+    # begin EBRAINS (added): add version
+    version("23_rc6", sha256="debb52e6ef2e1b4eaa917a8b4d4934bd2388659c660501a81ea044903bf9ee9d")
+    # end EBRAINS
+    version("22jlmrcc", sha256="1571d4e0f7d45b2a71dce5999fa875aea8c90ee219eb218d7916bf30ea229121")
 
+    depends_on("flex", type="build")
+    depends_on("bison", type="build")
+    depends_on("tcsh", type="build")
+    depends_on("zlib", type=("build", "run"))
+    depends_on("bzip2", type=("build", "run"))
+    depends_on("blas", type=("build", "run"))
+    depends_on("lapack", type=("build", "run"))
+    depends_on("arpack-ng", type=("build", "run"))
+    depends_on("netcdf-c", type=("build", "run"))
+    depends_on("netcdf-fortran", type=("build", "run"))
+    depends_on("fftw", type=("build", "run"))
+    depends_on("readline", type=("build", "run"))
+    depends_on("netlib-xblas~plain_blas", type=("build", "run"))
+    # Specific variants needed for boost according to build logs
+    depends_on(
+        "boost+thread+system+program_options+iostreams+regex+timer+chrono+filesystem+graph",
+        type=("build", "run"),
+    )
     # Python dependencies
-    # WARNING: If a python 3.8 version is already installed in spack then the '+tkinter' variant makes spack ignore the version
-    # WARNING: Spack may try to install the preferred python version (i.e. python 3.10.8)
-    # WARNING: The soultion is uninstall python and reinstall with this variant
-    depends_on('python@3.8: +tkinter',      type=('build', 'run'))
-    depends_on("py-numpy",                  type=("build", "run"))
-    depends_on("py-matplotlib",             type=("build", "run"))
-    depends_on("py-scipy",                  type=("build", "run"))
+    # begin EBRAINS (modified): add version
+    depends_on("python@3.8:3.10 +tkinter", type=("build", "run"), when="@22jlmrcc")
+    depends_on("python@3.8: +tkinter", type=("build", "run"), when="@23_rc6")
+    # end EBRAINS
+    depends_on("py-setuptools", type="build")
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-matplotlib", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
 
     def cmake_args(self):
         # Translated from ambertools build/run_cmake script
-        # We also add the TRUST_SYSTEM_LIBS argument that is mentioned in the ambertools CMake guide
+        # We also add the TRUST_SYSTEM_LIBS argument mentioned in the ambertools guide
         #   https://ambermd.org/pmwiki/pmwiki.php/Main/CMake-Guide-to-Options
         args = [
             self.define("COMPILER", "GNU"),
@@ -58,9 +64,9 @@ class Ambertools (CMakePackage):
             self.define("DOWNLOAD_MINICONDA", False),
             self.define("TRUST_SYSTEM_LIBS", True),
             # This is to avoid the x11 (X11_Xext_LIB) error
-            # It is equivalent to the '-noX11' flag accoridng to the docs:
+            # It is equivalent to the "-noX11" flag accoridng to the docs:
             # https://ambermd.org/pmwiki/pmwiki.php/Main/CMake-Common-Options
-            self.define("BUILD_GUI", False)
+            self.define("BUILD_GUI", False),
         ]
         return args
 
@@ -72,7 +78,17 @@ class Ambertools (CMakePackage):
         env.set("AMBER_PREFIX", self.prefix)
         env.set("AMBERHOME", self.prefix)
 
-    @run_after('install')
+    @run_after("install")
     @on_package_attributes(run_tests=True)
     def check_install(self):
         make("test.serial")
+
+    # Temporarily copy netcdf.h header file to netcdf-fortran/include to pass the Ambertools
+    # cmake check (quickest fix, will probably cause problems, needs to change)
+    @run_before("cmake")
+    def fix_check(self):
+        cp = Executable("cp")
+        cp(
+            self.spec["netcdf-c"].headers.directories[0] + "/netcdf.h",
+            self.spec["netcdf-fortran"].headers.directories[0],
+        )

packages/apbs/package.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-from spack import *
+from spack.package import *
 
 
 class Apbs(CMakePackage):
@@ -38,9 +38,9 @@ class Apbs(CMakePackage):
     depends_on('boost',             type=('build', 'run'))
     depends_on('blas',              type=('build', 'run'))
     depends_on('arpack-ng',         type=('build', 'run'))
-    depends_on('suite-sparse+tbb',  type=('build', 'run'))
+    depends_on('suite-sparse',      type=('build', 'run'))
     depends_on('maloc',             type=('build', 'run'))
-    depends_on('python@3.8:3.10',   type=('build', 'run'))
+    depends_on('python@3.8:3.11',   type=('build', 'run'))
 
     def cmake_args(self):
         # Min and max Python versions need to be set as variables to pass tests.

packages/arbor/package.py

@@ -13,10 +13,15 @@ class Arbor(CMakePackage, CudaPackage):
     homepage = "https://arbor-sim.org"
     git = "https://github.com/arbor-sim/arbor.git"
     url = "https://github.com/arbor-sim/arbor/releases/download/v0.9.0/arbor-v0.9.0-full.tar.gz"
-    maintainers = ["thorstenhater", "brenthuisman", "haampie"]
+    maintainers = ("thorstenhater", "ErbB4", "haampie")
 
-    version("master", branch="master")
-    version("develop")
+    version("master", branch="master", submodules=True)
+    version("develop", branch="master", submodules=True)
+    version(
+        "0.10.0",
+        sha256="72966b7a2f45ce259b8ba167ca3e4f5ab9f212136a300267aaac0c04ed3fe3fc",
+        url="https://github.com/arbor-sim/arbor/releases/download/v0.10.1/arbor-v0.10.0-full.tar.gz",
+    )
     version(
         "0.9.0",
         sha256="5f9740955c821aca81e23298c17ad64f33f635756ad9b4a0c1444710f564306a",
@@ -75,36 +80,50 @@ class Arbor(CMakePackage, CudaPackage):
     conflicts("%cce@:9.1")
     conflicts("%intel")
 
+    # begin EBRAINS (modified: added run dep)
     depends_on("cmake@3.19:", type=("build", "run"))
+    # end EBRAINS
 
     # misc dependencies
     depends_on("fmt@7.1:", when="@0.5.3:")  # required by the modcc compiler
     depends_on("fmt@9.1:", when="@0.7.1:")
-    depends_on("googletest@1.12.1", when="@0.7.1:")
+    # begin EBRAINS (modified: relaxed (upstream gave no info about update))
+    # upstream adds: depends_on("fmt@10.1:", when="@0.9.1:")
+    depends_on("googletest@1.12.1:", when="@0.7.1:")
     depends_on("pugixml@1.11:", when="@0.7.1:")
-    depends_on("nlohmann-json@3.11.2")
+    # upstream adds: depends_on("pugixml@1.13:", when="@0.9.1:")
+    depends_on("nlohmann-json@3.11.2:")
     depends_on("random123")
+    #upstream adds: depends_on("random123@1.14.0:", when="@0.10:")
+    # end EBRAINS (modified)
     with when("+cuda"):
         depends_on("cuda@10:")
         depends_on("cuda@11:", when="@0.7.1:")
+        depends_on("cuda@12:", when="@0.9.1:")
 
     # mpi
+    # begin EBRAINS (modified: added run dep)
     depends_on("mpi", when="+mpi", type=("build", "run"))
+    # end EBRAINS (modified)
     depends_on("py-mpi4py", when="+mpi+python", type=("build", "run"))
 
     # python (bindings)
-    extends("python", when="+python")
-    depends_on("python@3.7:", when="+python", type=("build", "run"))
-    depends_on("py-numpy", when="+python", type=("build", "run"))
     with when("+python"):
+        extends("python")
+        depends_on("python@3.7:", type=("build", "run"))
+        depends_on("python@3.9:", when="@0.9.1:", type=("build", "run"))
+        depends_on("py-numpy", type=("build", "run"))
         depends_on("py-pybind11@2.6:", type="build")
         depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type="build")
         depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type="build")
+        depends_on("py-pandas", type="test")
+        depends_on("py-seaborn", type="test")
 
     # sphinx based documentation
-    depends_on("python@3.7:", when="+doc", type="build")
-    depends_on("py-sphinx", when="+doc", type="build")
-    depends_on("py-svgwrite", when="+doc", type="build")
+    with when("+doc"):
+        depends_on("python@3.10:", type="build")
+        depends_on("py-sphinx", type="build")
+        depends_on("py-svgwrite", type="build")
 
     @property
     def build_targets(self):
@@ -124,10 +143,14 @@ class Arbor(CMakePackage, CudaPackage):
 
         # query spack for the architecture-specific compiler flags set by its wrapper
         args.append("-DARB_ARCH=none")
-        opt_flags = self.spec.target.optimization_flags(
-            self.spec.compiler.name, self.spec.compiler.version
+        opt_flags = spack.build_environment.optimization_flags(
+            self.compiler, self.spec.target
         )
-        args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags)
+        # Might return nothing
+        if opt_flags:
+            args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags)
+        # Needed, spack has no units package
+        args.append("-DARB_USE_BUNDLED_UNITS=ON")
 
         return args
 
@@ -135,3 +158,4 @@ class Arbor(CMakePackage, CudaPackage):
     @on_package_attributes(run_tests=True)
     def install_test(self):
         python("-c", "import arbor")
+        python("python/example/single_cell_model.py")

packages/bazel/bazelconfiguration-0.3.patch

---- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
-+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
-@@ -150,6 +150,13 @@
-       builder.put("PATH", null);
-       builder.put("LD_LIBRARY_PATH", null);
-     }
-+
-+    Map<String, String> spackEnv = System.getenv();
-+    for (String envName : spackEnv.keySet()) {
-+      if (envName.startsWith("SPACK_")) {
-+        builder.put(envName, spackEnv.get(envName));
-+      }
-+    }
-   }
- 
-   private static PathFragment determineShellExecutable(OS os, PathFragment fromOption) {

packages/bazel/bazelruleclassprovider-0.14.patch

---- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
-+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
-@@ -168,6 +168,13 @@ public class BazelRuleClassProvider {
-       env.put("PATH", null);
-     }
- 
-+    Map<String, String> spackEnv = System.getenv();
-+    for (String envName : spackEnv.keySet()) {
-+      if (envName.startsWith("SPACK_")) {
-+        env.put(envName, spackEnv.get(envName));
-+      }
-+    }
-+
-     // Shell environment variables specified via options take precedence over the
-     // ones inherited from the fragments. In the long run, these fragments will
-     // be replaced by appropriate default rc files anyway.

packages/bazel/build-0.29.1.patch

-From 9c9d27561780bc56d9f0867e325c7421a94ee1cb Mon Sep 17 00:00:00 2001
-From: Harsh Bhatia <bhatia4@llnl.gov>
-Date: Tue, 15 Dec 2020 15:56:10 -0800
-Subject: [PATCH] https://github.com/bazelbuild/bazel/commit/ab62a6e097590dac5ec946ad7a796ea0e8593ae0 
-
----
- src/conditions/BUILD | 6 ++++++
- third_party/BUILD    | 8 ++++++--
- 2 files changed, 12 insertions(+), 2 deletions(-)
-
-diff --git a/src/conditions/BUILD b/src/conditions/BUILD
-index 2b28e28057..faa41a439d 100644
---- a/src/conditions/BUILD
-+++ b/src/conditions/BUILD
-@@ -10,6 +10,12 @@ filegroup(
-     visibility = ["//src:__pkg__"],
- )
- 
-+config_setting(
-+    name = "linux_ppc",
-+    values = {"cpu": "ppc"},
-+    visibility = ["//visibility:public"],
-+)
-+
- config_setting(
-     name = "linux_x86_64",
-     values = {"cpu": "k8"},
-diff --git a/third_party/BUILD b/third_party/BUILD
-index 159006d741..4fcae54c00 100644
---- a/third_party/BUILD
-+++ b/third_party/BUILD
-@@ -523,12 +523,13 @@ UNNECESSARY_DYNAMIC_LIBRARIES = select({
-     "//src/conditions:darwin": "*.so *.dll",
-     "//src/conditions:darwin_x86_64": "*.so *.dll",
-     "//src/conditions:linux_x86_64": "*.jnilib *.dll",
-+    "//src/conditions:linux_ppc": "*.so *.jnilib *.dll",
-     # The .so file is an x86 one, so we can just remove it if the CPU is not x86
-     "//src/conditions:arm": "*.so *.jnilib *.dll",
-     "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll",
-     # Play it safe -- better have a big binary than a slow binary
-     # zip -d does require an argument. Supply something bogus.
--    "//conditions:default": "*.bogusextension",
-+    "//conditions:default": "",
- })
- 
- # Remove native libraries that are for a platform different from the one we are
-@@ -537,7 +538,10 @@ genrule(
-     name = "filter_netty_dynamic_libs",
-     srcs = ["netty_tcnative/netty-tcnative-boringssl-static-2.0.24.Final.jar"],
-     outs = ["netty_tcnative/netty-tcnative-filtered.jar"],
--    cmd = "cp $< $@ && zip -qd $@ " + UNNECESSARY_DYNAMIC_LIBRARIES,
-+    cmd = "cp $< $@ && " +
-+      # End successfully if there is nothing to be deleted from the archive
-+      "if [ -n '" + UNNECESSARY_DYNAMIC_LIBRARIES + "' ]; then " +
-+      "zip -qd $@ " + UNNECESSARY_DYNAMIC_LIBRARIES + "; fi",
- )
- 
- java_import(
--- 
-2.21.0 (Apple Git-122.2)
-

packages/bazel/cc_configure-0.3.0.patch

---- a/tools/cpp/cc_configure.bzl
-+++ b/tools/cpp/cc_configure.bzl
-@@ -173,8 +173,19 @@
-   else:
-     inc_dirs = result.stderr[index1 + 1:index2].strip()
- 
--  return [repository_ctx.path(_cxx_inc_convert(p))
--          for p in inc_dirs.split("\n")]
-+  default_inc_directories = [
-+    repository_ctx.path(_cxx_inc_convert(p))
-+    for p in inc_dirs.split("\n")
-+  ]
-+
-+  env = repository_ctx.os.environ
-+  if "SPACK_INCLUDE_DIRS" in env:
-+    for path in env["SPACK_INCLUDE_DIRS"].split(":"):
-+      default_inc_directories.append(
-+        repository_ctx.path(_cxx_inc_convert(path))
-+      )
-+
-+  return default_inc_directories
- 
- def _add_option_if_supported(repository_ctx, cc, option):
-   """Checks that `option` is supported by the C compiler."""

packages/bazel/cc_configure-0.5.0.patch

---- a/tools/cpp/cc_configure.bzl
-+++ b/tools/cpp/cc_configure.bzl
-@@ -200,8 +200,19 @@
-   else:
-     inc_dirs = result.stderr[index1 + 1:index2].strip()
- 
--  return [_escape_string(repository_ctx.path(_cxx_inc_convert(p)))
--          for p in inc_dirs.split("\n")]
-+  default_inc_directories = [
-+    _escape_string(repository_ctx.path(_cxx_inc_convert(p)))
-+    for p in inc_dirs.split("\n")
-+  ]
-+
-+  env = repository_ctx.os.environ
-+  if "SPACK_INCLUDE_DIRS" in env:
-+    for path in env["SPACK_INCLUDE_DIRS"].split(":"):
-+      default_inc_directories.append(
-+        repository_ctx.path(_cxx_inc_convert(path))
-+      )
-+
-+  return default_inc_directories
- 
- 
- def _add_option_if_supported(repository_ctx, cc, option):

packages/bazel/compile-0.13.patch

---- a/compile.sh
-+++ b/compile.sh
-@@ -92,7 +92,7 @@
- log "Building output/bazel"
- # We set host and target platform directly since the defaults in @bazel_tools
- # have not yet been generated.
--bazel_build "src:bazel${EXE_EXT}" \
-+CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel${EXE_EXT}" \
-   --host_platform=@bazel_tools//platforms:host_platform \
-   --platforms=@bazel_tools//platforms:target_platform \
-   || fail "Could not build Bazel"

packages/bazel/compile-0.16.patch

---- a/compile.sh
-+++ b/compile.sh
-@@ -92,7 +92,7 @@ display "."
- log "Building output/bazel"
- # We set host and target platform directly since the defaults in @bazel_tools
- # have not yet been generated.
--bazel_build "src:bazel_nojdk${EXE_EXT}" \
-+CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel_nojdk${EXE_EXT}" \
-   --host_platform=@bazel_tools//platforms:host_platform \
-   --platforms=@bazel_tools//platforms:target_platform \
-   || fail "Could not build Bazel"