From 500e8be3ccda2f34e5df82f4c0d160d9b714cb3d Mon Sep 17 00:00:00 2001
From: Eleni Mathioulaki <emathioulaki@athenarc.gr>
Date: Mon, 3 Jul 2023 20:02:02 +0200
Subject: [PATCH] replace awk script with spack-python
---
.gitlab-ci.yml | 4 +-
install_spack_env.sh | 3 +-
site-config/ymerge.py | 14 +------
site-config/ymerge.sh | 89 -------------------------------------------
4 files changed, 5 insertions(+), 105 deletions(-)
delete mode 100644 site-config/ymerge.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3343fcfa..753ec194 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,7 +199,7 @@ build-spack-env-on-runner:
- spack env create $SPACK_DEV_ENV
- spack env activate $SPACK_DEV_ENV
- rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
- - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+ - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
- spack install -y -j2 --fresh --test root
after_script:
- mkdir spack_logs
@@ -274,7 +274,7 @@ sync-gitlab-spack-instance:
# update environment site-configs
- rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
# update spack.yaml: merge top-level and site-specific spack.yaml files
- - bash site-config/ymerge.sh spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+ - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
# There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested
# in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command:
- rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file"
diff --git a/install_spack_env.sh b/install_spack_env.sh
index 95cddb7c..58c5c6e1 100644
--- a/install_spack_env.sh
+++ b/install_spack_env.sh
@@ -59,8 +59,7 @@ rm -rf $SPACK_ENV/site-config && cp -r $EBRAINS_REPO/site-config $SPACK_ENV
# update spack.yaml: merge top-level and site-specific spack.yaml files
# TODO: find a more robust way to do this (maybe with yq? but we need to download the binary)
# TODO: or add pyyaml to dependencies and use python script
-# python3 $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
-bash $EBRAINS_REPO/site-config/ymerge.sh $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
+spack-python $EBRAINS_REPO/site-config/ymerge.py $EBRAINS_REPO/spack.yaml $EBRAINS_REPO/site-config/$SYSTEMNAME/spack.yaml > $SPACK_ENV/spack.yaml
# There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies
# for installation tests. The workaround suggested in the issue is to NOT concretize separately, but simply
diff --git a/site-config/ymerge.py b/site-config/ymerge.py
index d515f69e..23a1f0a6 100644
--- a/site-config/ymerge.py
+++ b/site-config/ymerge.py
@@ -1,14 +1,4 @@
-import yaml
import sys
+from spack.config import merge_yaml, read_config_file, syaml
-with open(sys.argv[1], 'r') as f:
- env = yaml.load(f, yaml.SafeLoader)
-with open(sys.argv[2], 'r') as f:
- site_env = yaml.load(f, yaml.SafeLoader)
-
-for k in site_env['spack'].keys():
- if k in env['spack'].keys():
- if isinstance(env['spack'][k], list): env['spack'][k].extend(site_env['spack'][k])
- else: env['spack'][k] = site_env['spack'][k]
-
-yaml.dump(env, sys.stdout)
+print(syaml.dump(merge_yaml(read_config_file(sys.argv[1]), read_config_file(sys.argv[2]))))
diff --git a/site-config/ymerge.sh b/site-config/ymerge.sh
deleted file mode 100644
index 5e09e287..00000000
--- a/site-config/ymerge.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/sh
-
-flatten='BEGIN {
- KEYSEP = "\000"
-}
-
-BEGINFILE {
- level = 0
- indent = 0
-}
-
-/^ *#/ {
- next
-}
-
-/^ *-/ {
- sub(/^ */, "", $0)
- sub(/^- */, "- ", $0)
- print level_to_key[level] KEYSEP $0
- next
-}
-
-/:/ {
- match($0, /^ *([^ ]+:) *(.*)/, parts)
- key = parts[1]
- value = parts[2]
- this_indent = get_indent($0)
- if (this_indent < indent && ! (this_indent in indent_to_level)) {
- print("Malformed YAML") > (/dev/) stderr
- exit 1
- }
- indent = this_indent
- if (indent in indent_to_level) {
- level = indent_to_level[indent]
- } else {
- indent_to_level[indent] = ++level
- }
- level_to_key[level] = level > 1 ? level_to_key[level - 1] KEYSEP key : key
- if (value != "") {
- print level_to_key[level] KEYSEP value
- }
- next
-}
-
-
-function get_indent(line)
-{
- n = length(line)
- sub(/^ */, "", line)
- return (n - length(line))
-}
-'
-
-pprint='BEGIN {
- FS = "\000"
- level = 0
- BASE_INDENT = " "
-}
-
-{
- for (i = 1; i < NF; i++) {
- if ($i != prev_key[i]) {
- break
- }
- }
- indent = ""
- for (j = 1; j < i; j++) {
- indent = indent BASE_INDENT
- }
- for (; i < NF - 1; i++) {
- print indent $i
- indent = indent BASE_INDENT
- prev_key[i] = $i
- }
- if ($NF ~ /^- /) {
- if (prev_key[NF - 1] != $(NF - 1)) {
- print indent $(NF - 1)
- print indent BASE_INDENT $NF
- } else {
- print indent $NF
- }
- } else {
- print indent $(NF - 1), $NF
- }
- prev_key[NF - 1] = $(NF - 1)
-}
-'
-
-awk "$flatten" "$@" | sort -u | awk "$pprint"
--
GitLab