diff --git a/.gitignore b/.gitignore index 8d35cb3277ff6eb0d637147c09e95934604a431c..9266bb6183571e66a620d16be1f7a07ce3efe499 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ __pycache__ *.pyc +*.err +*.out +.spack-env/ +spack.lock diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb69e4e9baf873871e299e2cb39732dca83e9e70..c75c3094ab440a4b249185721a35c621b7cf0ad8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ variables: GITLAB_BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:devel SPACK_VERSION: v0.19.2 SPACK_PATH_GITLAB: /mnt/spack_v0.19.2 + SYSTEMNAME: ebrainslab # start an OpenShift Job that will build the Spack environment .deploy-build-environment: @@ -181,7 +182,6 @@ build-spack-env-on-runner: SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack script: - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH - - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs. # quick fix for that: (TODO: investigate more and open PR) - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_DEV_PATH/lib/spack/llnl/util/tty/log.py @@ -196,8 +196,11 @@ build-spack-env-on-runner: - spack load gcc@10.3.0 - spack compiler find - spack repo add . - - spack env create $SPACK_DEV_ENV spack.yaml + - spack env create $SPACK_DEV_ENV - spack env activate $SPACK_DEV_ENV + - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV + - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml + - cp /tmp/spack.yaml $SPACK_ENV/ - spack install -y -j2 --fresh --test root after_script: - mkdir spack_logs @@ -253,8 +256,6 @@ sync-gitlab-spack-instance: # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs. # quick fix for that: (TODO: investigate more and open PR) - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_PATH/spack/lib/spack/llnl/util/tty/log.py - # copy package settings - - cp $CI_PROJECT_DIR/packages.yaml $SPACK_PATH/spack/etc/spack/packages.yaml # activate spack - . $SPACK_PATH/spack/share/spack/setup-env.sh # get latest state of EBRAINS repo @@ -262,16 +263,20 @@ sync-gitlab-spack-instance: - spack repo list | grep -q ebrains-spack-builds && echo "Repository registered already" || spack repo add $SPACK_REPO_PATH - spack repo list # install compiler (if not already installed) - - EBRAINS_SPACK_COMPILER=$(grep 'compiler' $CI_PROJECT_DIR/packages.yaml | awk -F'[][]' '{ print $2 }') + - EBRAINS_SPACK_COMPILER=$(grep 'compiler' $CI_PROJECT_DIR/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }') - spack compiler find - spack install $EBRAINS_SPACK_COMPILER - spack load $EBRAINS_SPACK_COMPILER - spack compiler find - spack compiler list - # activate and update environment (and create it, if it doesn't exist) - - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV $CI_PROJECT_DIR/spack.yaml + # activate environment (and create it, if it doesn't exist) + - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV - spack env activate $SPACK_NFS_ENV - - cp $CI_PROJECT_DIR/spack.yaml $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.yaml + # update environment site-configs + - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV + # update spack.yaml: merge top-level and site-specific spack.yaml files + - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml + - cp /tmp/spack.yaml $SPACK_ENV/ # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command: - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file" diff --git a/README.md b/README.md index 6531d9140044e61886e9303592574aa012812bb1..3170a8a94a93a09373c6ac62548a8e3c41f416ba 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ This repository is where the EBRAINS Spack packages are maintained. It contains: - the latest EBRAINS [Spack environment](https://spack.readthedocs.io/en/latest/environments.html) ([spack.yaml](spack.yaml)), a list of the specific versions of EBRAINS tools that are part of the software release - a script ([install_spack_env.sh](install_spack_env.sh)) that can be used to install the latest spack environment on any machine (either install from scratch or update an existing installation) - CI/CD pipelines configured for (a) testing the installation of new packages and (b) deploying the environment to the EBRAINS Collaboratory Lab. +- (WIP) all the site-specific configurations that have been used to install the EBRAINS Spack environment in different systems, such as the EBRAINS Lab and the Fenix HPC systems. ## Contributing @@ -42,9 +43,8 @@ Clone Spack. We currently use version v0.18.1: git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch v0.18.1 https://github.com/spack/spack ``` -Modify (if needed) and copy the package.yaml file (that contains important package settings for Spack) and activate Spack: +Activate Spack: ``` -cp ebrains-spack-builds/packages.yaml spack/etc/spack/packages.yaml . spack/share/spack/setup-env.sh ``` @@ -53,14 +53,21 @@ Add the project repository to your Spack environment: spack repo add ebrains-spack-builds ``` -Create and activate the environment: +Create the environment: ``` -spack env create ebrains-env ebrains-spack-builds/spack.yaml -spack env activate ebrains-env +spack env create -d ebrains-spack-builds/ ``` -Install: +Define your site-specific configurations: ``` +export SYSTEMNAME=<your-system-name> +mkdir ebrains-spack-builds/site-config/$SYSTEMNAME +# copy any site-specific .yaml files inside the new dir +``` + +Activate the environment and install: +``` +spack env activate ebrains-spack-builds spack install --fresh ``` diff --git a/create_job.sh b/create_job.sh index e8bc5fca02ef692b4d68aa13d503b6389b564c41..dffe9131df8b0c817004541dbc5669f60699d374 100644 --- a/create_job.sh +++ b/create_job.sh @@ -39,6 +39,8 @@ spec: mountPath: /srv command: ["/usr/local/bin/deploy-build-env.sh", "$OP", "$INSTALLATION_ROOT", "$SPACK_VERSION", "$SPACK_ENV", "$BRANCH", "$RELEASE_NAME", "$LAB_KERNEL_ROOT"] env: + - name: SYSTEMNAME + value: ebrainslab - name: GITLAB_USER valueFrom: secretKeyRef: diff --git a/install_spack_env.sh b/install_spack_env.sh index b782ac2355dd75dd451b14a55d67ba2505a655b3..16311c4712d228d9a0dfc79e9f0c5ae9a934e12c 100644 --- a/install_spack_env.sh +++ b/install_spack_env.sh @@ -17,6 +17,9 @@ EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack +# define SYSTEMNAME variable in sites where it's not already defined +export SYSTEMNAME=${SYSTEMNAME:-${HPC_SYSTEM:-$BSC_MACHINE}} + # initial setup: clone spack if spack dir doesn't already exist if [ ! -d $INSTALLATION_ROOT/spack ] then @@ -26,15 +29,18 @@ then sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $INSTALLATION_ROOT/spack/lib/spack/llnl/util/tty/log.py fi -# copy package settings (modify if necessary) and activate Spack -cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml +# activate Spack source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh # install platform compiler (extract version from packages.yaml) -EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }') -spack compiler find -spack install $EBRAINS_SPACK_COMPILER -spack load $EBRAINS_SPACK_COMPILER +if [ $SYSTEMNAME == ebrainslab ] +then + EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }') + spack compiler find + spack install $EBRAINS_SPACK_COMPILER + spack load $EBRAINS_SPACK_COMPILER +fi + spack compiler find # add repo if it does not exist @@ -49,12 +55,17 @@ then spack env create $EBRAINS_SPACK_ENV fi -# update and activate environment -cp $EBRAINS_REPO/spack.yaml $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.yaml +# activate environment spack env activate $EBRAINS_SPACK_ENV +# update environment site-configs +rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV +# update spack.yaml: merge top-level and site-specific spack.yaml files +spack-python $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml +cp /tmp/spack.yaml $SPACK_ENV/ + # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies # for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply -# remove the .lock file and let the enironment be concretized by the spack install command: +# remove the .lock file and let the environment be concretized by the spack install command: rm $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.lock || echo "No spack.lock file" # install the environment, use 2 jobs to reduce the amount of required RAM spack install -y -j2 --fresh --test root diff --git a/site-config/.gitignore b/site-config/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1e47f841169f3344c84e8b8934c8887e32a303af --- /dev/null +++ b/site-config/.gitignore @@ -0,0 +1,10 @@ +# exclude all files to allow sites to add own configs +*/ + +# un-exclude the configs tracked in this repository +!ebrainslab +!g100 +!jurecadc +!jusuf +!hsmcompute + diff --git a/site-config/README.md b/site-config/README.md new file mode 100644 index 0000000000000000000000000000000000000000..46d8ae711839c00cb905c5b70ecc627ccb06bfac --- /dev/null +++ b/site-config/README.md @@ -0,0 +1,21 @@ +# Site specific config for Spack + +Usually the machine name is available in an environment variable like +`$SYSTEMNAME`, `$HPC_SYSTEM` or equivalent to load local base configuration. +The environment base `spack.yaml` then includes all files in the specific +folder: + +```yaml +spack: + include: + - site-config/$SYSTEMNAME +``` + +You can check the overlay is working by +```bash +spack env activate . +spack config blame config +``` + +Some system specific entries should be governed by the site-specific +envrionment config. diff --git a/packages.yaml b/site-config/ebrainslab/packages.yaml similarity index 100% rename from packages.yaml rename to site-config/ebrainslab/packages.yaml diff --git a/site-config/ebrainslab/spack.yaml b/site-config/ebrainslab/spack.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce36e8bbbe52a8bf9fab41b283972e2415b0cdc9 --- /dev/null +++ b/site-config/ebrainslab/spack.yaml @@ -0,0 +1,13 @@ +spack: + specs: + # Notebook + - py-ipycanvas + - py-ipykernel + - py-ipython + - py-notebook + - r-irkernel + # "collab"-specific constraint to match ("jupyterlab_widgets") in the base image + - py-ipywidgets@:7.7 + # Collab utils + - clb-nb-utils@0.1.0 + diff --git a/site-config/g100/.gitkeep b/site-config/g100/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/site-config/hsmcompute/.gitkeep b/site-config/hsmcompute/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/site-config/jurecadc/compilers.yaml b/site-config/jurecadc/compilers.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69485034782d471a353dad0e6fd9768e9a23936f --- /dev/null +++ b/site-config/jurecadc/compilers.yaml @@ -0,0 +1,27 @@ +compilers: +- compiler: + spec: gcc@11.3.0 + paths: + cc: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gcc + cxx: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/g++ + f77: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gfortran + fc: /p/software/jurecadc/stages/2023/software/GCCcore/11.3.0/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: null + fc: null + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/site-config/jurecadc/config.yaml b/site-config/jurecadc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dd3b0b0fc3d377809a5833bcb3142d4c418b285 --- /dev/null +++ b/site-config/jurecadc/config.yaml @@ -0,0 +1,10 @@ +config: + build_stage: + - $USERSOFTWARE/spack/user-cache/$user/spack-stage + - $tempdir/$user/spack-stage + source_cache: $USERSOFTWARE/spack/source-cache + test_stage: $USERSOFTWARE/spack/user-cache/$user/test + misc_cache: $USERSOFTWARE/spack/user-cache/$user/cache + install_tree: + root: $USERSOFTWARE/install + db_lock_timeout: 10 diff --git a/site-config/jurecadc/packages.yaml b/site-config/jurecadc/packages.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5f285aa75a09d7905cccc1545bdd55b29970c30 --- /dev/null +++ b/site-config/jurecadc/packages.yaml @@ -0,0 +1,115 @@ +# `spack external find` with +# Currently Loaded Modules: +# 1) Stages/2023 (S) 2) GCCcore/.11.3.0 (H) 3) zlib/.1.2.12 (H) 4) binutils/.2.38 (H) 5) StdEnv/2023 6) spack/0.19.2 +packages: + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + git: + externals: + - spec: git@2.31.1~tcltk + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /p/software/jurecadc/stages/2023/software/binutils/2.38-GCCcore-11.3.0 + - spec: binutils@2.30.117 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + curl: + externals: + - spec: curl@7.61.1+gssapi+ldap+nghttp2 + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@2.13.3 + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr +# packages: +# all: +# compiler: [gcc@11.2.0] +# python: +# buildable: False +# version: [3.9.6] +# externals: +# - spec: "python@3.9.6%gcc@11.2.0 arch=linux-rocky8-zen2" +# prefix: /p/software/jurecadc/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/ +# openssl: +# buildable: False +# version: [1.1.1k] +# externals: +# - spec: openssl@1.1.1k +# prefix: /p/software/jurecadc/stages/2022/software/OpenSSL/1.1/ +# mpi: +# buildable: False +# openmpi: +# externals: +# - spec: "openmpi@4.1.2%gcc@11.2.0 arch=linux-rocky8-zen2" +# prefix: /p/software/jurecadc/stages/2022/software/OpenMPI/4.1.2-GCC-11.2.0/ diff --git a/site-config/jusuf/.gitkeep b/site-config/jusuf/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/site-config/ymerge.py b/site-config/ymerge.py new file mode 100644 index 0000000000000000000000000000000000000000..accb877977fd1a95b83ea3ec44179d7928059ae8 --- /dev/null +++ b/site-config/ymerge.py @@ -0,0 +1,14 @@ +# spack-python script that merges two environment configuration files (spack.yaml) into one +# Usage: spack-python /path/to/first/spack.yaml /path/to/second/spack.yaml +# (note: if the second file does not exist, the output is the first file + +import sys, os +from spack.config import merge_yaml, read_config_file, syaml + +if not os.path.exists(sys.argv[2]): + merged = syaml.dump(read_config_file(sys.argv[1])) +else: + merged = syaml.dump(merge_yaml(read_config_file(sys.argv[1]), read_config_file(sys.argv[2]))) + +print(merged) + diff --git a/spack.yaml b/spack.yaml index 1a29f7fb40fea94d87eacdbee78c837740d167ef..8a0ada0e0bf074f01e7e1abf7e9c982b616cf09c 100644 --- a/spack.yaml +++ b/spack.yaml @@ -1,15 +1,7 @@ spack: + include: + - site-config/$SYSTEMNAME specs: - # Notebook - - py-ipycanvas - - py-ipykernel - - py-ipython - - py-notebook - - r-irkernel - # "collab"-specific constraint to match ("jupyterlab_widgets") in the base image - - py-ipywidgets@:7.7 - # Collab utils - - clb-nb-utils@0.1.0 # EBRAINS tools - apbs@3.4.0 - arbor@0.8.1 +python +mpi