diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4530f98ae6cfd539ecc02c3d3f17d31b7a2bba9b..691e9fea3c678df430d399ba8bed2b206118669c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -182,7 +182,6 @@ build-spack-env-on-runner: SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack script: - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH - - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs. # quick fix for that: (TODO: investigate more and open PR) - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_DEV_PATH/lib/spack/llnl/util/tty/log.py @@ -197,9 +196,10 @@ build-spack-env-on-runner: - spack load gcc@10.3.0 - spack compiler find - spack repo add . - - spack env create $SPACK_DEV_ENV spack.yaml - - cp -r site-config $SPACK_ROOT/var/spack/environments/$SPACK_DEV_ENV/ + - spack env create $SPACK_DEV_ENV - spack env activate $SPACK_DEV_ENV + - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV + - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml - spack install -y -j2 --fresh --test root after_script: - mkdir spack_logs @@ -255,8 +255,6 @@ sync-gitlab-spack-instance: # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs. # quick fix for that: (TODO: investigate more and open PR) - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_PATH/spack/lib/spack/llnl/util/tty/log.py - # copy package settings - - cp $CI_PROJECT_DIR/packages.yaml $SPACK_PATH/spack/etc/spack/packages.yaml # activate spack - . $SPACK_PATH/spack/share/spack/setup-env.sh # get latest state of EBRAINS repo @@ -270,10 +268,13 @@ sync-gitlab-spack-instance: - spack load $EBRAINS_SPACK_COMPILER - spack compiler find - spack compiler list - # activate and update environment (and create it, if it doesn't exist) - - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV $CI_PROJECT_DIR/spack.yaml - - cp -r -t $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/ $CI_PROJECT_DIR/{site-config,spack.yaml} + # activate environment (and create it, if it doesn't exist) + - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV - spack env activate $SPACK_NFS_ENV + # update environment site-configs + - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV + # update spack.yaml: merge top-level and site-specific spack.yaml files + - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command: - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file" diff --git a/install_spack_env.sh b/install_spack_env.sh index 6106b6733c371760472b8520e6d45fc8a3a3e2d5..95cddb7c63534fafb85eb03b29d49751b368bfac 100644 --- a/install_spack_env.sh +++ b/install_spack_env.sh @@ -26,15 +26,18 @@ then sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $INSTALLATION_ROOT/spack/lib/spack/llnl/util/tty/log.py fi -# copy package settings (modify if necessary) and activate Spack -cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml +# activate Spack source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh # install platform compiler (extract version from packages.yaml) -EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }') -spack compiler find -spack install $EBRAINS_SPACK_COMPILER -spack load $EBRAINS_SPACK_COMPILER +if [ $SYSTEMNAME == ebrainslab ] +then + EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }') + spack compiler find + spack install $EBRAINS_SPACK_COMPILER + spack load $EBRAINS_SPACK_COMPILER +fi + spack compiler find # add repo if it does not exist @@ -49,12 +52,19 @@ then spack env create $EBRAINS_SPACK_ENV fi -# update and activate environment -cp -r -t $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/ $EBRAINS_REPO/{site-config,spack.yaml} +# activate environment spack env activate $EBRAINS_SPACK_ENV +# update environment site-configs +rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV +# update spack.yaml: merge top-level and site-specific spack.yaml files +# TODO: find a more robust way to do this (maybe with yq? but we need to download the binary) +# TODO: or add pyyaml to dependencies and use python script +# python3 $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml +bash $EBRAINS_REPO/site-config/ymerge.sh $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml + # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies # for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply -# remove the .lock file and let the enironment be concretized by the spack install command: +# remove the .lock file and let the environment be concretized by the spack install command: rm $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.lock || echo "No spack.lock file" # install the environment, use 2 jobs to reduce the amount of required RAM spack install -y -j2 --fresh --test root diff --git a/site-config/ymerge.py b/site-config/ymerge.py new file mode 100644 index 0000000000000000000000000000000000000000..d515f69efffb9630604433552d8803f5853c2e75 --- /dev/null +++ b/site-config/ymerge.py @@ -0,0 +1,14 @@ +import yaml +import sys + +with open(sys.argv[1], 'r') as f: + env = yaml.load(f, yaml.SafeLoader) +with open(sys.argv[2], 'r') as f: + site_env = yaml.load(f, yaml.SafeLoader) + +for k in site_env['spack'].keys(): + if k in env['spack'].keys(): + if isinstance(env['spack'][k], list): env['spack'][k].extend(site_env['spack'][k]) + else: env['spack'][k] = site_env['spack'][k] + +yaml.dump(env, sys.stdout) diff --git a/site-config/ymerge.sh b/site-config/ymerge.sh new file mode 100644 index 0000000000000000000000000000000000000000..5e09e2872b93bcc1998e254be543a5e8c7600b63 --- /dev/null +++ b/site-config/ymerge.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +flatten='BEGIN { + KEYSEP = "\000" +} + +BEGINFILE { + level = 0 + indent = 0 +} + +/^ *#/ { + next +} + +/^ *-/ { + sub(/^ */, "", $0) + sub(/^- */, "- ", $0) + print level_to_key[level] KEYSEP $0 + next +} + +/:/ { + match($0, /^ *([^ ]+:) *(.*)/, parts) + key = parts[1] + value = parts[2] + this_indent = get_indent($0) + if (this_indent < indent && ! (this_indent in indent_to_level)) { + print("Malformed YAML") > (/dev/) stderr + exit 1 + } + indent = this_indent + if (indent in indent_to_level) { + level = indent_to_level[indent] + } else { + indent_to_level[indent] = ++level + } + level_to_key[level] = level > 1 ? level_to_key[level - 1] KEYSEP key : key + if (value != "") { + print level_to_key[level] KEYSEP value + } + next +} + + +function get_indent(line) +{ + n = length(line) + sub(/^ */, "", line) + return (n - length(line)) +} +' + +pprint='BEGIN { + FS = "\000" + level = 0 + BASE_INDENT = " " +} + +{ + for (i = 1; i < NF; i++) { + if ($i != prev_key[i]) { + break + } + } + indent = "" + for (j = 1; j < i; j++) { + indent = indent BASE_INDENT + } + for (; i < NF - 1; i++) { + print indent $i + indent = indent BASE_INDENT + prev_key[i] = $i + } + if ($NF ~ /^- /) { + if (prev_key[NF - 1] != $(NF - 1)) { + print indent $(NF - 1) + print indent BASE_INDENT $NF + } else { + print indent $NF + } + } else { + print indent $(NF - 1), $NF + } + prev_key[NF - 1] = $(NF - 1) +} +' + +awk "$flatten" "$@" | sort -u | awk "$pprint"