replace awk script with spack-python

.gitlab-ci.yml

@@ -187,7 +187,7 @@ build-spack-env-on-runner:
     - spack env create $SPACK_DEV_ENV
     - spack env activate $SPACK_DEV_ENV
     - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
-    - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
     - spack install -y -j2 --fresh --test root
   after_script:
     - mkdir spack_logs
@@ -264,7 +264,7 @@ sync-gitlab-spack-instance:
     # update environment site-configs
     - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
     # update spack.yaml: merge top-level and site-specific spack.yaml files
-    - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
     # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested
     # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command:
     - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file"

install_spack_env.sh

@@ -58,8 +58,7 @@ rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV
 # update spack.yaml: merge top-level and site-specific spack.yaml files
 # TODO: find a more robust way to do this (maybe with yq? but we need to download the binary)
 # TODO: or add pyyaml to dependencies and use python script
-# python3 $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
-bash $EBRAINS_REPO/site-config/ymerge.sh $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+spack-python $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
 
 # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies
 # for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply

site-config/ymerge.py

-import yaml
 import sys
+from spack.config import merge_yaml, read_config_file, syaml
 
-with open(sys.argv[1], 'r') as f:
-  env = yaml.load(f, yaml.SafeLoader)
-with open(sys.argv[2], 'r') as f:
-  site_env = yaml.load(f, yaml.SafeLoader)
-
-for k in site_env['spack'].keys():
-  if k in env['spack'].keys():
-  	if isinstance(env['spack'][k], list): env['spack'][k].extend(site_env['spack'][k])
-  else: env['spack'][k] = site_env['spack'][k]
-
-yaml.dump(env, sys.stdout)
+print(syaml.dump(merge_yaml(read_config_file(sys.argv[1]), read_config_file(sys.argv[2]))))

site-config/ymerge.sh

-#!/bin/sh
-
-flatten='BEGIN {
-	KEYSEP = "\000"
-}
-
-BEGINFILE {
-	level = 0
-	indent = 0
-}
-
-/^ *#/ {
-	next
-}
-
-/^ *-/ {
-	sub(/^ */, "", $0)
-	sub(/^- */, "- ", $0)
-	print level_to_key[level] KEYSEP $0
-	next
-}
-
-/:/ {
-	match($0, /^ *([^ ]+:) *(.*)/, parts)
-	key = parts[1]
-	value = parts[2]
-	this_indent = get_indent($0)
-	if (this_indent < indent && ! (this_indent in indent_to_level)) {
-		print("Malformed YAML") > (/dev/) stderr
-		exit 1
-	}
-	indent = this_indent
-	if (indent in indent_to_level) {
-		level = indent_to_level[indent]
-	} else {
-		indent_to_level[indent] = ++level
-	}
-	level_to_key[level] = level > 1 ? level_to_key[level - 1] KEYSEP key : key
-	if (value != "") {
-		print level_to_key[level] KEYSEP value
-	}
-	next
-}
-
-
-function get_indent(line)
-{
-	n = length(line)
-	sub(/^ */, "", line)
-	return (n - length(line))
-}
-'
-
-pprint='BEGIN {
-	FS = "\000"
-	level = 0
-	BASE_INDENT = "  "
-}
-
-{
-	for (i = 1; i < NF; i++) {
-		if ($i != prev_key[i]) {
-			break
-		}
-	}
-	indent = ""
-	for (j = 1; j < i; j++) {
-		indent = indent BASE_INDENT
-	}
-	for (; i < NF - 1; i++) {
-		print indent $i
-		indent = indent BASE_INDENT
-		prev_key[i] = $i
-	}
-	if ($NF ~ /^- /) {
-		if (prev_key[NF - 1] != $(NF - 1)) {
-			print indent $(NF - 1)
-			print indent BASE_INDENT $NF
-		} else {
-			print indent $NF
-		}
-	} else {
-		print indent $(NF - 1), $NF
-	}
-	prev_key[NF - 1] = $(NF - 1)
-}
-'
-
-awk "$flatten" "$@" | sort -u | awk "$pprint"