Merge branch 'add-site-specific-configs' into 'master'

Add site specific configs

See merge request technical-coordination/project-internal/devops/platform/ebrains-spack-builds!367

.gitignore

 __pycache__
 *.pyc
+*.err
+*.out
+.spack-env/
+spack.lock

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
   GITLAB_BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:devel
   SPACK_VERSION: v0.19.2
   SPACK_PATH_GITLAB: /mnt/spack_v0.19.2
+  SYSTEMNAME: ebrainslab
 
 # start an OpenShift Job that will build the Spack environment
 .deploy-build-environment:
@@ -181,7 +182,6 @@ build-spack-env-on-runner:
     SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack
   script:
     - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH
-    - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml
     # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
     # quick fix for that: (TODO: investigate more and open PR)
     - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_DEV_PATH/lib/spack/llnl/util/tty/log.py
@@ -196,8 +196,11 @@ build-spack-env-on-runner:
     - spack load gcc@10.3.0
     - spack compiler find
     - spack repo add .
-    - spack env create $SPACK_DEV_ENV spack.yaml
+    - spack env create $SPACK_DEV_ENV
     - spack env activate $SPACK_DEV_ENV
+    - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
+    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
+    - cp /tmp/spack.yaml $SPACK_ENV/
     - spack install -y -j2 --fresh --test root
   after_script:
     - mkdir spack_logs
@@ -253,8 +256,6 @@ sync-gitlab-spack-instance:
     # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
     # quick fix for that: (TODO: investigate more and open PR)
     - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_PATH/spack/lib/spack/llnl/util/tty/log.py
-    # copy package settings
-    - cp $CI_PROJECT_DIR/packages.yaml $SPACK_PATH/spack/etc/spack/packages.yaml
     # activate spack
     - . $SPACK_PATH/spack/share/spack/setup-env.sh
     # get latest state of EBRAINS repo
@@ -262,16 +263,20 @@ sync-gitlab-spack-instance:
     - spack repo list | grep -q ebrains-spack-builds && echo "Repository registered already" || spack repo add $SPACK_REPO_PATH
     - spack repo list
     # install compiler (if not already installed)
-    - EBRAINS_SPACK_COMPILER=$(grep 'compiler' $CI_PROJECT_DIR/packages.yaml | awk -F'[][]' '{ print $2 }')
+    - EBRAINS_SPACK_COMPILER=$(grep 'compiler' $CI_PROJECT_DIR/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }')
     - spack compiler find
     - spack install $EBRAINS_SPACK_COMPILER
     - spack load $EBRAINS_SPACK_COMPILER
     - spack compiler find
     - spack compiler list
-    # activate and update environment (and create it, if it doesn't exist)
-    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV $CI_PROJECT_DIR/spack.yaml
+    # activate environment (and create it, if it doesn't exist)
+    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV
     - spack env activate $SPACK_NFS_ENV
-    - cp $CI_PROJECT_DIR/spack.yaml $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.yaml
+    # update environment site-configs
+    - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
+    # update spack.yaml: merge top-level and site-specific spack.yaml files
+    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
+    - cp /tmp/spack.yaml $SPACK_ENV/
     # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested
     # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command:
     - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file"

README.md

@@ -15,6 +15,7 @@ This repository is where the EBRAINS Spack packages are maintained. It contains:
 - the latest EBRAINS [Spack environment](https://spack.readthedocs.io/en/latest/environments.html) ([spack.yaml](spack.yaml)), a list of the specific versions of EBRAINS tools that are part of the software release
 - a script ([install_spack_env.sh](install_spack_env.sh)) that can be used to install the latest spack environment on any machine (either install from scratch or update an existing installation)
 - CI/CD pipelines configured for (a) testing the installation of new packages and (b) deploying the environment to the EBRAINS Collaboratory Lab.
+- (WIP) all the site-specific configurations that have been used to install the EBRAINS Spack environment in different systems, such as the EBRAINS Lab and the Fenix HPC systems.
 
 ## Contributing
 
@@ -42,9 +43,8 @@ Clone Spack. We currently use version v0.18.1:
 git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch v0.18.1 https://github.com/spack/spack
 ```
 
-Modify (if needed) and copy the package.yaml file (that contains important package settings for Spack) and activate Spack:
+Activate Spack:
 ```
-cp ebrains-spack-builds/packages.yaml spack/etc/spack/packages.yaml
 . spack/share/spack/setup-env.sh
 ```
 
@@ -53,14 +53,21 @@ Add the project repository to your Spack environment:
 spack repo add ebrains-spack-builds
 ```
 
-Create and activate the environment:
+Create the environment:
 ```
-spack env create ebrains-env ebrains-spack-builds/spack.yaml
-spack env activate ebrains-env
+spack env create -d ebrains-spack-builds/
 ```
 
-Install:
+Define your site-specific configurations:
 ```
+export SYSTEMNAME=<your-system-name>
+mkdir ebrains-spack-builds/site-config/$SYSTEMNAME
+# copy any site-specific .yaml files inside the new dir
+```
+
+Activate the environment and install:
+```
+spack env activate ebrains-spack-builds
 spack install --fresh
 ```
 

create_job.sh

@@ -39,6 +39,8 @@ spec:
             mountPath: /srv
         command: ["/usr/local/bin/deploy-build-env.sh", "$OP", "$INSTALLATION_ROOT", "$SPACK_VERSION", "$SPACK_ENV", "$BRANCH", "$RELEASE_NAME", "$LAB_KERNEL_ROOT"]
         env:
+          - name: SYSTEMNAME
+            value: ebrainslab
           - name: GITLAB_USER
             valueFrom:
               secretKeyRef:

install_spack_env.sh

@@ -17,6 +17,9 @@ EBRAINS_SPACK_ENV=$4  # name of EBRAINS Spack environment to be created/updated
 export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
 export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
 
+# define SYSTEMNAME variable in sites where it's not already defined
+export SYSTEMNAME=${SYSTEMNAME:-${HPC_SYSTEM:-$BSC_MACHINE}}
+
 # initial setup: clone spack if spack dir doesn't already exist
 if [ ! -d $INSTALLATION_ROOT/spack ]
 then
@@ -26,15 +29,18 @@ then
   sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $INSTALLATION_ROOT/spack/lib/spack/llnl/util/tty/log.py
 fi
 
-# copy package settings (modify if necessary) and activate Spack
-cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml
+# activate Spack
 source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
 
 # install platform compiler (extract version from packages.yaml)
-EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }')
-spack compiler find
-spack install $EBRAINS_SPACK_COMPILER
-spack load $EBRAINS_SPACK_COMPILER
+if [ $SYSTEMNAME == ebrainslab ]
+then
+  EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }')
+  spack compiler find
+  spack install $EBRAINS_SPACK_COMPILER
+  spack load $EBRAINS_SPACK_COMPILER
+fi
+
 spack compiler find
 
 # add repo if it does not exist
@@ -49,12 +55,17 @@ then
   spack env create $EBRAINS_SPACK_ENV
 fi
 
-# update and activate environment
-cp $EBRAINS_REPO/spack.yaml $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.yaml
+# activate environment
 spack env activate $EBRAINS_SPACK_ENV
+# update environment site-configs
+rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV
+# update spack.yaml: merge top-level and site-specific spack.yaml files
+spack-python $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
+cp /tmp/spack.yaml $SPACK_ENV/
+
 # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies
 # for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply
-# remove the .lock file and let the enironment be concretized by the spack install command:
+# remove the .lock file and let the environment be concretized by the spack install command:
 rm $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.lock || echo "No spack.lock file"
 # install the environment, use 2 jobs to reduce the amount of required RAM
 spack install -y -j2 --fresh --test root

site-config/.gitignore

+# exclude all files to allow sites to add own configs
+*/
+
+# un-exclude the configs tracked in this repository
+!ebrainslab
+!g100
+!jurecadc
+!jusuf
+!hsmcompute
+

site-config/README.md

+# Site specific config for Spack
+
+Usually the machine name is available in an environment variable like
+`$SYSTEMNAME`, `$HPC_SYSTEM` or equivalent to load local base configuration.
+The environment base `spack.yaml` then includes all files in the specific
+folder:
+
+```yaml
+spack:
+  include:
+  - site-config/$SYSTEMNAME
+```
+
+You can check the overlay is working by
+```bash
+spack env activate .
+spack config blame config
+```
+
+Some system specific entries should be governed by the site-specific
+envrionment config.

site-config/ebrainslab/spack.yaml

+spack:
+  specs:
+    # Notebook
+    - py-ipycanvas
+    - py-ipykernel
+    - py-ipython
+    - py-notebook
+    - r-irkernel
+    # "collab"-specific constraint to match ("jupyterlab_widgets") in the base image
+    - py-ipywidgets@:7.7
+    # Collab utils
+    - clb-nb-utils@0.1.0
+

site-config/jurecadc/compilers.yaml

+compilers:
+- compiler:
+    spec: gcc@11.3.0
+    paths:
+      cc: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gcc
+      cxx: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/g++
+      f77: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gfortran
+      fc: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gfortran
+    flags: {}
+    operating_system: rocky8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: gcc@8.5.0
+    paths:
+      cc: /usr/bin/gcc
+      cxx: /usr/bin/g++
+      f77: null
+      fc: null
+    flags: {}
+    operating_system: rocky8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

site-config/jurecadc/config.yaml

+config:
+  build_stage:
+  - $USERSOFTWARE/spack/user-cache/$user/spack-stage
+  - $tempdir/$user/spack-stage
+  source_cache: $USERSOFTWARE/spack/source-cache
+  test_stage: $USERSOFTWARE/spack/user-cache/$user/test
+  misc_cache: $USERSOFTWARE/spack/user-cache/$user/cache
+  install_tree:
+    root: $USERSOFTWARE/install
+  db_lock_timeout: 10

site-config/jurecadc/packages.yaml

+# `spack external find` with
+# Currently Loaded Modules:
+#   1) Stages/2023 (S)   2) GCCcore/.11.3.0 (H)   3) zlib/.1.2.12 (H)   4) binutils/.2.38 (H)   5) StdEnv/2023   6) spack/0.19.2
+packages:
+  findutils:
+    externals:
+    - spec: findutils@4.6.0
+      prefix: /usr
+  groff:
+    externals:
+    - spec: groff@1.22.3
+      prefix: /usr
+  flex:
+    externals:
+    - spec: flex@2.6.1+lex
+      prefix: /usr
+  diffutils:
+    externals:
+    - spec: diffutils@3.6
+      prefix: /usr
+  git:
+    externals:
+    - spec: git@2.31.1~tcltk
+      prefix: /usr
+  m4:
+    externals:
+    - spec: m4@1.4.18
+      prefix: /usr
+  texinfo:
+    externals:
+    - spec: texinfo@6.5
+      prefix: /usr
+  subversion:
+    externals:
+    - spec: subversion@1.10.2
+      prefix: /usr
+  libtool:
+    externals:
+    - spec: libtool@2.4.6
+      prefix: /usr
+  binutils:
+    externals:
+    - spec: binutils@2.38
+      prefix: /p/software/jurecadc/stages/2023/software/binutils/2.38-GCCcore-11.3.0
+    - spec: binutils@2.30.117
+      prefix: /usr
+  openssl:
+    externals:
+    - spec: openssl@1.1.1k
+      prefix: /usr
+  pkgconf:
+    externals:
+    - spec: pkgconf@1.4.2
+      prefix: /usr
+  automake:
+    externals:
+    - spec: automake@1.16.1
+      prefix: /usr
+  tar:
+    externals:
+    - spec: tar@1.30
+      prefix: /usr
+  gmake:
+    externals:
+    - spec: gmake@4.2.1
+      prefix: /usr
+  openssh:
+    externals:
+    - spec: openssh@8.0p1
+      prefix: /usr
+  curl:
+    externals:
+    - spec: curl@7.61.1+gssapi+ldap+nghttp2
+      prefix: /usr
+  git-lfs:
+    externals:
+    - spec: git-lfs@2.13.3
+      prefix: /usr
+  gawk:
+    externals:
+    - spec: gawk@4.2.1
+      prefix: /usr
+  coreutils:
+    externals:
+    - spec: coreutils@8.30
+      prefix: /usr
+  bison:
+    externals:
+    - spec: bison@3.0.4
+      prefix: /usr
+  autoconf:
+    externals:
+    - spec: autoconf@2.69
+      prefix: /usr
+#  packages:
+#    all:
+#      compiler: [gcc@11.2.0]
+#    python:
+#      buildable: False
+#      version: [3.9.6]
+#      externals:
+#        - spec: "python@3.9.6%gcc@11.2.0 arch=linux-rocky8-zen2"
+#          prefix: /p/software/jurecadc/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/
+#    openssl:
+#      buildable: False
+#      version: [1.1.1k]
+#      externals:
+#        - spec: openssl@1.1.1k
+#          prefix: /p/software/jurecadc/stages/2022/software/OpenSSL/1.1/
+#    mpi:
+#      buildable: False
+#    openmpi:
+#      externals:
+#        - spec: "openmpi@4.1.2%gcc@11.2.0 arch=linux-rocky8-zen2"
+#          prefix: /p/software/jurecadc/stages/2022/software/OpenMPI/4.1.2-GCC-11.2.0/

site-config/ymerge.py

+# spack-python script that merges two environment configuration files (spack.yaml) into one
+# Usage: spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml
+# (note: if the second file does not exist, the output is the first file
+
+import sys, os
+from spack.config import merge_yaml, read_config_file, syaml
+
+if not os.path.exists(sys.argv[2]):
+    merged = syaml.dump(read_config_file(sys.argv[1]))
+else:
+    merged = syaml.dump(merge_yaml(read_config_file(sys.argv[1]), read_config_file(sys.argv[2])))
+
+print(merged)
+

spack.yaml

 spack:
+  include:
+    - site-config/$SYSTEMNAME
   specs:
-    # Notebook
-    - py-ipycanvas
-    - py-ipykernel
-    - py-ipython
-    - py-notebook
-    - r-irkernel
-    # "collab"-specific constraint to match ("jupyterlab_widgets") in the base image
-    - py-ipywidgets@:7.7
-    # Collab utils
-    - clb-nb-utils@0.1.0
     # EBRAINS tools
     - apbs@3.4.0
     - arbor@0.8.1 +python +mpi