update deployment scripts to merge spack.yaml files

.gitlab-ci.yml

@@ -182,7 +182,6 @@ build-spack-env-on-runner:
     SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack
   script:
     - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH
-    - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml
     # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
     # quick fix for that: (TODO: investigate more and open PR)
     - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_DEV_PATH/lib/spack/llnl/util/tty/log.py
@@ -197,9 +196,10 @@ build-spack-env-on-runner:
     - spack load gcc@10.3.0
     - spack compiler find
     - spack repo add .
-    - spack env create $SPACK_DEV_ENV spack.yaml
-    - cp -r site-config $SPACK_ROOT/var/spack/environments/$SPACK_DEV_ENV/
+    - spack env create $SPACK_DEV_ENV
     - spack env activate $SPACK_DEV_ENV
+    - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
+    - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
     - spack install -y -j2 --fresh --test root
   after_script:
     - mkdir spack_logs
@@ -255,8 +255,6 @@ sync-gitlab-spack-instance:
     # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
     # quick fix for that: (TODO: investigate more and open PR)
     - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_PATH/spack/lib/spack/llnl/util/tty/log.py
-    # copy package settings
-    - cp $CI_PROJECT_DIR/packages.yaml $SPACK_PATH/spack/etc/spack/packages.yaml
     # activate spack
     - . $SPACK_PATH/spack/share/spack/setup-env.sh
     # get latest state of EBRAINS repo
@@ -270,10 +268,13 @@ sync-gitlab-spack-instance:
     - spack load $EBRAINS_SPACK_COMPILER
     - spack compiler find
     - spack compiler list
-    # activate and update environment (and create it, if it doesn't exist)
-    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV $CI_PROJECT_DIR/spack.yaml
-    - cp -r -t $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/ $CI_PROJECT_DIR/{site-config,spack.yaml}
+    # activate environment (and create it, if it doesn't exist)
+    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV
     - spack env activate $SPACK_NFS_ENV
+    # update environment site-configs
+    - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
+    # update spack.yaml: merge top-level and site-specific spack.yaml files
+    - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
     # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested
     # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command:
     - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file"

install_spack_env.sh

@@ -26,15 +26,18 @@ then
   sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $INSTALLATION_ROOT/spack/lib/spack/llnl/util/tty/log.py
 fi
 
-# copy package settings (modify if necessary) and activate Spack
-cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml
+# activate Spack
 source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
 
 # install platform compiler (extract version from packages.yaml)
-EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }')
-spack compiler find
-spack install $EBRAINS_SPACK_COMPILER
-spack load $EBRAINS_SPACK_COMPILER
+if [ $SYSTEMNAME == ebrainslab ]
+then
+  EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }')
+  spack compiler find
+  spack install $EBRAINS_SPACK_COMPILER
+  spack load $EBRAINS_SPACK_COMPILER
+fi
+
 spack compiler find
 
 # add repo if it does not exist
@@ -49,12 +52,19 @@ then
   spack env create $EBRAINS_SPACK_ENV
 fi
 
-# update and activate environment
-cp -r -t $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/ $EBRAINS_REPO/{site-config,spack.yaml}
+# activate environment
 spack env activate $EBRAINS_SPACK_ENV
+# update environment site-configs
+rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV
+# update spack.yaml: merge top-level and site-specific spack.yaml files
+# TODO: find a more robust way to do this (maybe with yq? but we need to download the binary)
+# TODO: or add pyyaml to dependencies and use python script
+# python3 $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+bash $EBRAINS_REPO/site-config/ymerge.sh $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+
 # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies
 # for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply
-# remove the .lock file and let the enironment be concretized by the spack install command:
+# remove the .lock file and let the environment be concretized by the spack install command:
 rm $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/spack.lock || echo "No spack.lock file"
 # install the environment, use 2 jobs to reduce the amount of required RAM
 spack install -y -j2 --fresh --test root

site-config/ymerge.py

+import yaml
+import sys
+
+with open(sys.argv[1], 'r') as f:
+  env = yaml.load(f, yaml.SafeLoader)
+with open(sys.argv[2], 'r') as f:
+  site_env = yaml.load(f, yaml.SafeLoader)
+
+for k in site_env['spack'].keys():
+  if k in env['spack'].keys():
+  	if isinstance(env['spack'][k], list): env['spack'][k].extend(site_env['spack'][k])
+  else: env['spack'][k] = site_env['spack'][k]
+
+yaml.dump(env, sys.stdout)

site-config/ymerge.sh

+#!/bin/sh
+
+flatten='BEGIN {
+	KEYSEP = "\000"
+}
+
+BEGINFILE {
+	level = 0
+	indent = 0
+}
+
+/^ *#/ {
+	next
+}
+
+/^ *-/ {
+	sub(/^ */, "", $0)
+	sub(/^- */, "- ", $0)
+	print level_to_key[level] KEYSEP $0
+	next
+}
+
+/:/ {
+	match($0, /^ *([^ ]+:) *(.*)/, parts)
+	key = parts[1]
+	value = parts[2]
+	this_indent = get_indent($0)
+	if (this_indent < indent && ! (this_indent in indent_to_level)) {
+		print("Malformed YAML") > (/dev/) stderr
+		exit 1
+	}
+	indent = this_indent
+	if (indent in indent_to_level) {
+		level = indent_to_level[indent]
+	} else {
+		indent_to_level[indent] = ++level
+	}
+	level_to_key[level] = level > 1 ? level_to_key[level - 1] KEYSEP key : key
+	if (value != "") {
+		print level_to_key[level] KEYSEP value
+	}
+	next
+}
+
+
+function get_indent(line)
+{
+	n = length(line)
+	sub(/^ */, "", line)
+	return (n - length(line))
+}
+'
+
+pprint='BEGIN {
+	FS = "\000"
+	level = 0
+	BASE_INDENT = "  "
+}
+
+{
+	for (i = 1; i < NF; i++) {
+		if ($i != prev_key[i]) {
+			break
+		}
+	}
+	indent = ""
+	for (j = 1; j < i; j++) {
+		indent = indent BASE_INDENT
+	}
+	for (; i < NF - 1; i++) {
+		print indent $i
+		indent = indent BASE_INDENT
+		prev_key[i] = $i
+	}
+	if ($NF ~ /^- /) {
+		if (prev_key[NF - 1] != $(NF - 1)) {
+			print indent $(NF - 1)
+			print indent BASE_INDENT $NF
+		} else {
+			print indent $NF
+		}
+	} else {
+		print indent $(NF - 1), $NF
+	}
+	prev_key[NF - 1] = $(NF - 1)
+}
+'
+
+awk "$flatten" "$@" | sort -u | awk "$pprint"