add script to post-fix generated TCL module files

operations/ensure_prepend.py

+#!/usr/bin/env python
+# encoding:utf8
+"""
+Filter to modify `setenv` statements in modulefiles to `prepend-path` statements.
+
+Usage:
+    tools/ensure_prepend.py PATH MANPATH PYTHONPATH <../build.52.VGk/test.mod
+or
+    module sh-to-module bash somescript.sh | tools/ensure_prepend.py PATH MANPATH PYTHONPATH > path/to/module
+"""
+
+import logging
+import os
+import re
+import sys
+
+logging.basicConfig(level=logging.DEBUG)
+log = logging.getLogger()
+
+def main():
+    names = sys.argv[1:]
+    log.debug("rewriting %s", names)
+
+    bad_re = re.compile(r'^setenv\s+(?P<varname>[a-zA-Z_0-9]+)\s+(?P<value>.*)')
+    for line in sys.stdin:
+        match = bad_re.match(line)
+        if not match or match.group('varname') not in names:
+            print(line, end='')
+            continue
+        log.debug("found bad line: %s", line.rstrip("\n"))
+        name = match.group("varname")
+        log.debug("name %s", name)
+        value = match.group("value")
+        log.debug("value %s", value)
+        parts = value.split(":")
+        log.debug("parts %s", parts)
+        have = os.environ.get(name, "").split(':')
+        log.debug("have %s", have)
+
+        print(f"prepend-path    {name} " + ":".join([p for p in parts if p not in have]))
+
+
+if __name__ == '__main__':
+    main()

operations/module2view.py

@@ -25,38 +25,111 @@ import re
 import sys
 import logging
 from pathlib import Path
+from collections import Counter
 
 SYSTEMNAME = os.environ.get("SYSTEMNAME", "jurecadc")
-screenwidth = int(os.environ.get("COLUMNS"))
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger()
 
-def rewrite(installpath, viewpath, value: str, separator=":") -> str:
+class Rewriter:
+    def __init__(self, installpath=None, viewpath=None):
+        self._installpath = installpath
+        self._viewpath = viewpath
+        self._export_re = re.compile(r"^export (?P<key>\w+)='?(?P<value>.*?)'?;?\n")
+        self._package_re = re.compile(r"(?P<installpath>.*)/(?P<package>\w[\w\d-]*?[\w\d])-(?P<version>[\d\w][\d\w\._]*)-(?P<hash>[0-9a-z]{32})/(?P<relative>.*)")
+
+    @property
+    def installpath(self):
+        return self._installpath
+
+    @property
+    def viewpath(self):
+        return self._viewpath
+
+    def __call__(self, line: str) -> str:
+        "Parse an input line and rewrite if necessary."
+
+        export = self._export_re.match(line.strip("'"))
+        if export:
+            key, value = export.groups()
+            log.debug("found export of %s", key)
+
+            if ":" in value:
+                value = self._rewrite(value, separator=":")
+            else:
+                value = self._rewrite(value, separator=" ")
+            return f"export {key}='{value[:]}'\n"
+        else:
+            log.debug("paste")
+            return line
+
+    def _rewrite(self, value: str, separator=":") -> str:
         """
         Change all spack install paths to the given viewpath and de-duplicate.
 
         Since not all variable values are collon-separated `:` like `$PATH`, the
         separator can optionally be changed (e.g. for `TCLLIBPATH`).
         """
-    package_re = re.compile(
-        r"(?P<package>\w[\w\d-]*?[\w\d])-(?P<version>[\d\w][\d\w\._]*)-(?P<hash>[0-9a-z]{32})/(?P<relative>.*)"
-    )
-    paths = []
+        log.debug("REWRITE value split by '%s'", separator)
+        log.debug("REWRITE value '%s'", value)
+
+        # heuristically use the first path to the view as viewpath
+        if self._viewpath is None:
+            possibles = [inpath for inpath in value.split(separator) if inpath.endswith(".spack-env/view")]
+            log.debug("possible paths: %s", possibles)
+            if possibles:
+                self._viewpath = possibles[0]
+                log.info("identifed viewpath: %s", self._viewpath)
+
+        if separator not in value:
+            log.debug("value paste (no separator)")
+            return value
+
+        if self._viewpath is None and "spack" in value:
+            log.warning("viewpath is still unknown")
+            log.debug("value paste (no viewpath)")
+            return value
+
+        inpaths = value.split(separator)
+        log.debug("inspecting %s paths", len(inpaths))
+
+        matches = [self._package_re.match(x).group("installpath") for x in value.split(separator) if self._package_re.match(x)]
+        log.debug("%d package_re matching installpaths: %s", len(matches), matches)
+
+        if not matches:
+            log.debug("no paths match")
+            return value
+        counts = Counter(matches)
+        log.debug("counts is %s", counts)
+        installpath = counts.most_common()[0][0]
+        log.debug("installpath is '%s'", installpath)
+
+        if self._viewpath is None:
+            raise ValueError("Found export but view path is not known yet")
+
+        outpaths = []
         for path in value.split(separator):
             if not path.startswith(installpath):
-            paths.append(path)
+                if path in outpaths:
+                    log.debug("already in output: %s", path)
+                    continue
+                log.debug("append unchanged:  %s", path)
+                outpaths.append(path)
                 continue
             subpath = path[len(installpath) :]  # match tail after installpath
-        toolpath = package_re.match(subpath)
+            toolpath = self._package_re.match(subpath)
             if toolpath:
                 if not re.match(r"\d", toolpath.group("version")):
                     log.warning(f"# !!! warning version not numeric in {toolpath.groups()} !!!\n#")
-            newpath = str(Path(viewpath) / toolpath.group("relative"))
-            if paths and paths[-1] != newpath:
-                paths.append(newpath)
+                newpath = str(Path(self._viewpath) / toolpath.group("relative"))
+                if newpath not in outpaths:
+                    log.debug("append   changed:  %s", newpath)
+                    outpaths.append(newpath)
+                else:
+                    log.debug("already in output: %s", newpath)
             else:
                 raise ValueError(f"could not decompose path {subpath}")
-    return separator.join(paths)
+        return separator.join(outpaths)
 
 
 def main():
@@ -64,27 +137,13 @@ def main():
     Convert all shell exports with paths to a Spack install directory to paths
     into an environment view.
     """
-    viewpath = None
-    installpath = f"/p/usersoftware/swmanage/EBRAINS/{SYSTEMNAME}/install/linux-rocky8-zen2/gcc-11.3.0/"  # py-setuptools-62.6.0-idjjqlikk266jeus6yoziulnifmnsjsi/lib/python3.10/site-packages"
-    log.debug(f"screen width: {screenwidth}")
+    # example installpaths
+    # /p/usersoftware/swmanage/EBRAINS/{SYSTEMNAME}/install/linux-rocky8-zen2/gcc-11.3.0/py-setuptools-62.6.0-idjjqlikk266jeus6yoziulnifmnsjsi/lib/python3.10/site-packages
 
-    export_re = re.compile(r"^export (?P<key>\w+)='?(?P<value>.*?)'?;?\n")
+    rewrite = Rewriter()
     with open(sys.argv[1], "r") as infile:
         for line in infile:
-            export = export_re.match(line.strip("'"))
-            if export:
-                key, value = export.groups()
-                if ":" not in value and value.endswith(".spack-env/view"):
-                    viewpath = value
-                    log.info(f"# identifed view path {viewpath}")
-                if viewpath and ":" in value:
-                    value = rewrite(installpath, viewpath, value, separator=":")
-                elif viewpath and " " in value:
-                    value = rewrite(installpath, viewpath, value, separator=" ")
-                # print(f"# export {key} = {len(value)} {value[:min(len(value), screenwidth - 20 - len(key))]}")
-                print(f"export {key}='{value[:]}'")
-            else:
-                print(line, end="")
+            sys.stdout.write(rewrite(line))
 
 
 if __name__ == "__main__":