From 244655b75fc01865bfca6d4a94e35716c1a4dce9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Fri, 8 Sep 2023 13:45:56 +0200
Subject: [PATCH 1/7] fix(wf-brainscales2-demos): connection to experiment
service
---
packages/wf-brainscales2-demos/package.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/wf-brainscales2-demos/package.py b/packages/wf-brainscales2-demos/package.py
index 74a3fcf7..4999678e 100644
--- a/packages/wf-brainscales2-demos/package.py
+++ b/packages/wf-brainscales2-demos/package.py
@@ -5,6 +5,7 @@
from spack import *
+import os
class WfBrainscales2Demos(Package):
@@ -43,6 +44,11 @@ class WfBrainscales2Demos(Package):
def installcheck(self):
cmd_env = which("env")
cmd_env()
+ # enable "EBRAINS lab" mode
+ os.environ["LAB_IMAGE_NAME"] = "EBRAINS"
+ # select "EBRAINS experimental" upstream experiment service
+ os.environ["LAB_KERNEL_NAME"] = "EBRAINS-experimental"
+ os.environ["JUPYTERHUB_USER"] = "spack-test-wf-brainscales2-demos"
# execute notebook and save
jupyter = Executable("jupyter")
jupyter("nbconvert",
--
GitLab
From a6162a642228dfcbcc880926129d5c2b0b10ad33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Tue, 12 Sep 2023 13:54:24 +0200
Subject: [PATCH 2/7] fix(BSS2): runtime/test dependency on oppulance
---
packages/hxtorch/package.py | 9 +++++----
packages/pynn-brainscales/package.py | 8 +++++---
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/packages/hxtorch/package.py b/packages/hxtorch/package.py
index 7be0e3ab..570796be 100644
--- a/packages/hxtorch/package.py
+++ b/packages/hxtorch/package.py
@@ -11,13 +11,14 @@ class Hxtorch(WafPackage):
"""hxtorch --- a PyTorch-based toplevel for the BrainScaleS-2 neuromorphic hardware systems"""
homepage = "https://github.com/electronicvisions/hxtorch"
- # This repo provides a waf binary used for the build below
+ # This repo provides a custom waf binary used for the build below
git = "https://github.com/electronicvisions/pynn-brainscales.git"
version('7.0-a1', branch='waf')
- # PPU compiler dependencies
- depends_on('oppulance@7.0-a1')
+ # compiler for the BrainScaleS-2 embedded processor ("PPU"); needed for
+ # building/linking, at runtime and for testing
+ depends_on('oppulance@7.0-a1', type=('build', 'link', 'run', 'test'))
# host software dependencies
depends_on('bitsery', type=('build', 'link', 'run'))
@@ -36,7 +37,7 @@ class Hxtorch(WafPackage):
depends_on('log4cxx@0.12.1:', type=('build', 'link', 'run'))
depends_on('pkgconfig', type=('build', 'link', 'run'))
depends_on('psmisc', type=('run', 'test'))
- depends_on('python@3.7.0:', type=('build', 'link', 'run')) # BrainScaleS(-2, type=('build', 'link', 'run')) only supports Python >= 3.7
+ depends_on('python@3.7.0:', type=('build', 'link', 'run')) # BrainScaleS-2 only supports Python >= 3.7
depends_on('py-h5py', type=('build', 'link', 'run')) # PyNN tests need it
depends_on('py-jax@0.3.25:', type=('build', 'link', 'run'))
depends_on('py-matplotlib', type=('build', 'link', 'run'))
diff --git a/packages/pynn-brainscales/package.py b/packages/pynn-brainscales/package.py
index 5c7d33f7..8b8a6735 100644
--- a/packages/pynn-brainscales/package.py
+++ b/packages/pynn-brainscales/package.py
@@ -11,12 +11,14 @@ class PynnBrainscales(WafPackage):
"""PyNN toplevel for the BrainScaleS-2 neuromorphic hardware systems"""
homepage = "https://github.com/electronicvisions/pynn-brainscales"
+ # This repo provides a custom waf binary used for the build below
git = "https://github.com/electronicvisions/pynn-brainscales.git"
version('7.0-a1', branch='waf')
- # PPU compiler dependencies
- depends_on('oppulance@7.0-a1')
+ # compiler for the BrainScaleS-2 embedded processor ("PPU"); needed for
+ # building/linking, at runtime and for testing
+ depends_on('oppulance@7.0-a1', type=('build', 'link', 'run', 'test'))
# host software dependencies
depends_on('bitsery', type=('build', 'link', 'run'))
@@ -35,7 +37,7 @@ class PynnBrainscales(WafPackage):
depends_on('log4cxx@0.12.1:', type=('build', 'link', 'run'))
depends_on('pkgconfig', type=('build', 'link', 'run'))
depends_on('psmisc', type=('run', 'test'))
- depends_on('python@3.7.0:', type=('build', 'link', 'run')) # BrainScaleS(-2, type=('build', 'link', 'run')) only supports Python >= 3.7
+ depends_on('python@3.7.0:', type=('build', 'link', 'run')) # BrainScaleS-2 only supports Python >= 3.7
depends_on('py-deap@1.3.1:', type=('build', 'link', 'run'))
depends_on('py-h5py', type=('build', 'link', 'run')) # PyNN tests need it
depends_on('py-matplotlib', type=('build', 'link', 'run'))
--
GitLab
From d03436210869e36bb877d4357fa12ed6b19003a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Wed, 13 Sep 2023 09:51:40 +0200
Subject: [PATCH 3/7] fix(BSS2): external test-time dependencies
---
packages/hxtorch/package.py | 8 ++++----
packages/pynn-brainscales/package.py | 6 +++---
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/packages/hxtorch/package.py b/packages/hxtorch/package.py
index 570796be..0077e4ed 100644
--- a/packages/hxtorch/package.py
+++ b/packages/hxtorch/package.py
@@ -21,15 +21,15 @@ class Hxtorch(WafPackage):
depends_on('oppulance@7.0-a1', type=('build', 'link', 'run', 'test'))
# host software dependencies
- depends_on('bitsery', type=('build', 'link', 'run'))
+ depends_on('bitsery', type=('build', 'link', 'run', 'test'))
depends_on('binutils+gold+ld+plugins', type=('build', 'link', 'run')) # specialize
- depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run')) # specialize boost (non-clingo, type=('build', 'link', 'run'))
+ depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run', 'test'))
depends_on('cereal', type=('build', 'link', 'run'))
depends_on('cppcheck', type=('build', 'link', 'run'))
depends_on('genpybind@ebrains', type=('build', 'link', 'run'))
depends_on('gflags', type=('build', 'link', 'run'))
depends_on('googletest@1.11.0:+gmock', type=('build', 'link', 'run')) # variadic templates needed
- depends_on('inja', type=('build', 'link', 'run')) # template engine for PPU source jit generation
+ depends_on('inja', type=('build', 'link', 'run', 'test')) # template engine for PPU source jit generation
depends_on('intel-tbb', type=('build', 'link', 'run')) # ppu gdbserver
depends_on('libelf', type=('build', 'link', 'run'))
depends_on('liblockfile', type=('build', 'link', 'run'))
@@ -49,7 +49,7 @@ class Hxtorch(WafPackage):
depends_on('py-pycodestyle', type=('build', 'link', 'run'))
depends_on('py-pyelftools', type=('build', 'link', 'run'))
depends_on('py-pylint', type=('build', 'link', 'run'))
- depends_on('py-torch@1.9.1:', type=('build', 'link', 'run'))
+ depends_on('py-torch@1.9.1:', type=('build', 'link', 'run', 'test'))
depends_on('py-torchvision', type=('run')) # for demos
depends_on('py-tree-math', type=('build', 'link', 'run'))
depends_on('py-pyyaml', type=('build', 'link', 'run'))
diff --git a/packages/pynn-brainscales/package.py b/packages/pynn-brainscales/package.py
index 8b8a6735..a6e58127 100644
--- a/packages/pynn-brainscales/package.py
+++ b/packages/pynn-brainscales/package.py
@@ -21,15 +21,15 @@ class PynnBrainscales(WafPackage):
depends_on('oppulance@7.0-a1', type=('build', 'link', 'run', 'test'))
# host software dependencies
- depends_on('bitsery', type=('build', 'link', 'run'))
+ depends_on('bitsery', type=('build', 'link', 'run', 'test'))
depends_on('binutils+gold+ld+plugins', type=('build', 'link', 'run')) # specialize
- depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run')) # specialize boost (non-clingo, type=('build', 'link', 'run'))
+ depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run', 'test'))
depends_on('cereal', type=('build', 'link', 'run'))
depends_on('cppcheck', type=('build', 'link', 'run'))
depends_on('genpybind@ebrains', type=('build', 'link', 'run'))
depends_on('gflags', type=('build', 'link', 'run'))
depends_on('googletest@1.11.0:+gmock', type=('build', 'link', 'run')) # variadic templates needed
- depends_on('inja', type=('build', 'link', 'run')) # template engine for PPU source jit generation
+ depends_on('inja', type=('build', 'link', 'run', 'test')) # template engine for PPU source jit generation
depends_on('intel-tbb', type=('build', 'link', 'run')) # ppu gdbserver
depends_on('libelf', type=('build', 'link', 'run'))
depends_on('liblockfile', type=('build', 'link', 'run'))
--
GitLab
From d91ddf3b976142251b22233d5a57adc67f17df0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Fri, 15 Sep 2023 09:12:59 +0200
Subject: [PATCH 4/7] fix(BSS2): provide include dirs for runtime ppu build
flow
---
packages/hxtorch/package.py | 21 +++++++++++++++++++--
packages/pynn-brainscales/package.py | 21 +++++++++++++++++++--
2 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/packages/hxtorch/package.py b/packages/hxtorch/package.py
index 0077e4ed..a470f65b 100644
--- a/packages/hxtorch/package.py
+++ b/packages/hxtorch/package.py
@@ -129,11 +129,28 @@ class Hxtorch(WafPackage):
env.set('WAF_CONFIGURE_LD_LIBRARY_PATH', ':'.join(library))
env.prepend_path('PATH', ':'.join(path))
- def setup_dependent_build_environment(self, env, dependent_spec):
+ def _setup_common_env(self, env):
+ # TODO: use standard install layout for Python modules and extensions'
+ # shared objects => remove then
env.prepend_path('PYTHONPATH', self.prefix.lib)
+ # grenade needs to find some libraries for the JIT-compilation of
+ # programs for BrainScaleS-2's embedded processor.
+ ppu_include_dirs = []
+ ppu_dep_names = ['bitsery', 'boost']
+ for ppu_dep_name in ppu_dep_names:
+ dep = self.spec[ppu_dep_name]
+ dep_include_dirs = set(dep.headers.directories)
+ print('ppu includes (', dep.name, '):', dep_include_dirs, "\n")
+ ppu_include_dirs.extend(list(dep_include_dirs))
+ env.set('C_INCLUDE_PATH', ':'.join(ppu_include_dirs))
+ env.set('CPLUS_INCLUDE_PATH', ':'.join(ppu_include_dirs))
+
+ def setup_dependent_build_environment(self, env, dependent_spec):
+ self._setup_common_env(env)
+
def setup_run_environment(self, env):
- env.prepend_path('PYTHONPATH', self.prefix.lib)
+ self._setup_common_env(env)
# override configure step as we perform a project setup first
def configure(self, spec, prefix):
diff --git a/packages/pynn-brainscales/package.py b/packages/pynn-brainscales/package.py
index a6e58127..ac532a84 100644
--- a/packages/pynn-brainscales/package.py
+++ b/packages/pynn-brainscales/package.py
@@ -106,11 +106,28 @@ class PynnBrainscales(WafPackage):
env.set('WAF_CONFIGURE_LD_LIBRARY_PATH', ':'.join(library))
env.prepend_path('PATH', ':'.join(path))
- def setup_dependent_build_environment(self, env, dependent_spec):
+ def _setup_common_env(self, env):
+ # TODO: use standard install layout for Python modules and extensions'
+ # shared objects => remove then
env.prepend_path('PYTHONPATH', self.prefix.lib)
+ # grenade needs to find some libraries for the JIT-compilation of
+ # programs for BrainScaleS-2's embedded processor.
+ ppu_include_dirs = []
+ ppu_dep_names = ['bitsery', 'boost']
+ for ppu_dep_name in ppu_dep_names:
+ dep = self.spec[ppu_dep_name]
+ dep_include_dirs = set(dep.headers.directories)
+ print('ppu includes (', dep.name, '):', dep_include_dirs, "\n")
+ ppu_include_dirs.extend(list(dep_include_dirs))
+ env.set('C_INCLUDE_PATH', ':'.join(ppu_include_dirs))
+ env.set('CPLUS_INCLUDE_PATH', ':'.join(ppu_include_dirs))
+
+ def setup_dependent_build_environment(self, env, dependent_spec):
+ self._setup_common_env(env)
+
def setup_run_environment(self, env):
- env.prepend_path('PYTHONPATH', self.prefix.lib)
+ self._setup_common_env(env)
# override configure step as we perform a project setup first
def configure(self, spec, prefix):
--
GitLab
From e744e05e5cc054d2daffa1289292e37fb8969f40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Fri, 15 Sep 2023 13:01:27 +0200
Subject: [PATCH 5/7] feat(wf-bss2-demos): add test_notebooks
---
packages/wf-brainscales2-demos/package.py | 27 ++++++++++++++++-------
1 file changed, 19 insertions(+), 8 deletions(-)
diff --git a/packages/wf-brainscales2-demos/package.py b/packages/wf-brainscales2-demos/package.py
index 4999678e..36d5c19f 100644
--- a/packages/wf-brainscales2-demos/package.py
+++ b/packages/wf-brainscales2-demos/package.py
@@ -39,16 +39,10 @@ class WfBrainscales2Demos(Package):
# sanity_check_prefix requires something in the install directory
mkdirp(prefix + "/.spack_test_results")
- @run_after("install")
- @on_package_attributes(run_tests=True)
- def installcheck(self):
+ def _run_notebooks(self):
+ # TODO: remove debug output
cmd_env = which("env")
cmd_env()
- # enable "EBRAINS lab" mode
- os.environ["LAB_IMAGE_NAME"] = "EBRAINS"
- # select "EBRAINS experimental" upstream experiment service
- os.environ["LAB_KERNEL_NAME"] = "EBRAINS-experimental"
- os.environ["JUPYTERHUB_USER"] = "spack-test-wf-brainscales2-demos"
# execute notebook and save
jupyter = Executable("jupyter")
jupyter("nbconvert",
@@ -60,3 +54,20 @@ class WfBrainscales2Demos(Package):
"ts_05-yin_yang.ipynb",
"--output",
prefix+"/.spack_test_results/ts_05-yin_yang.ipynb")
+
+ def _set_collab_things(self):
+ # enable "EBRAINS lab" mode
+ os.environ["LAB_IMAGE_NAME"] = "EBRAINS"
+ # select "EBRAINS experimental" upstream experiment service
+ os.environ["LAB_KERNEL_NAME"] = "EBRAINS-experimental"
+ os.environ["JUPYTERHUB_USER"] = "spack-test-wf-brainscales2-demos"
+
+ @run_after("install")
+ @on_package_attributes(run_tests=True)
+ def installcheck(self):
+ self._set_collab_things()
+ self._run_notebooks()
+
+ def test_notebooks(self):
+ self._set_collab_things()
+ self._run_notebooks()
--
GitLab
From 5b26f6f2e00ddedfda449912937d71bca61c129f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eric=20M=C3=BCller?= <mueller@kip.uni-heidelberg.de>
Date: Fri, 8 Sep 2023 13:44:32 +0200
Subject: [PATCH 6/7] feat(wf-brainscales2-demos): enable most notebooks
---
packages/wf-brainscales2-demos/package.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/packages/wf-brainscales2-demos/package.py b/packages/wf-brainscales2-demos/package.py
index 36d5c19f..69020d32 100644
--- a/packages/wf-brainscales2-demos/package.py
+++ b/packages/wf-brainscales2-demos/package.py
@@ -4,6 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from glob import glob
from spack import *
import os
@@ -45,15 +46,16 @@ class WfBrainscales2Demos(Package):
cmd_env()
# execute notebook and save
jupyter = Executable("jupyter")
- jupyter("nbconvert",
- "--ExecutePreprocessor.kernel_name=python3",
- "--execute",
- "--allow-errors",
- "--to",
- "notebook",
- "ts_05-yin_yang.ipynb",
- "--output",
- prefix+"/.spack_test_results/ts_05-yin_yang.ipynb")
+ for fn in glob("ts*.ipynb") + glob("tp*.ipynb"):
+ jupyter("nbconvert",
+ "--ExecutePreprocessor.kernel_name=python3",
+ "--execute",
+ "--allow-errors",
+ "--to",
+ "notebook",
+ fn,
+ "--output",
+ prefix + "/.spack_test_results/" + fn)
def _set_collab_things(self):
# enable "EBRAINS lab" mode
--
GitLab
From 6597838c0bb61fc308d89631c670fd56b467ec55 Mon Sep 17 00:00:00 2001
From: Eleni Mathioulaki <emathioulaki@athenarc.gr>
Date: Wed, 20 Sep 2023 13:34:40 +0200
Subject: [PATCH 7/7] also add cereal to ppu_include_dirs
---
packages/hxtorch/package.py | 4 ++--
packages/pynn-brainscales/package.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/packages/hxtorch/package.py b/packages/hxtorch/package.py
index a470f65b..137eebc6 100644
--- a/packages/hxtorch/package.py
+++ b/packages/hxtorch/package.py
@@ -24,7 +24,7 @@ class Hxtorch(WafPackage):
depends_on('bitsery', type=('build', 'link', 'run', 'test'))
depends_on('binutils+gold+ld+plugins', type=('build', 'link', 'run')) # specialize
depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run', 'test'))
- depends_on('cereal', type=('build', 'link', 'run'))
+ depends_on('cereal', type=('build', 'link', 'run', 'test'))
depends_on('cppcheck', type=('build', 'link', 'run'))
depends_on('genpybind@ebrains', type=('build', 'link', 'run'))
depends_on('gflags', type=('build', 'link', 'run'))
@@ -137,7 +137,7 @@ class Hxtorch(WafPackage):
# grenade needs to find some libraries for the JIT-compilation of
# programs for BrainScaleS-2's embedded processor.
ppu_include_dirs = []
- ppu_dep_names = ['bitsery', 'boost']
+ ppu_dep_names = ['bitsery', 'boost', 'cereal']
for ppu_dep_name in ppu_dep_names:
dep = self.spec[ppu_dep_name]
dep_include_dirs = set(dep.headers.directories)
diff --git a/packages/pynn-brainscales/package.py b/packages/pynn-brainscales/package.py
index ac532a84..2b05efb3 100644
--- a/packages/pynn-brainscales/package.py
+++ b/packages/pynn-brainscales/package.py
@@ -24,7 +24,7 @@ class PynnBrainscales(WafPackage):
depends_on('bitsery', type=('build', 'link', 'run', 'test'))
depends_on('binutils+gold+ld+plugins', type=('build', 'link', 'run')) # specialize
depends_on('boost@1.69.0: +graph+icu+mpi+numpy+coroutine+context+filesystem+python+serialization+system+thread+program_options cxxstd=17', type=('build', 'link', 'run', 'test'))
- depends_on('cereal', type=('build', 'link', 'run'))
+ depends_on('cereal', type=('build', 'link', 'run', 'test'))
depends_on('cppcheck', type=('build', 'link', 'run'))
depends_on('genpybind@ebrains', type=('build', 'link', 'run'))
depends_on('gflags', type=('build', 'link', 'run'))
@@ -114,7 +114,7 @@ class PynnBrainscales(WafPackage):
# grenade needs to find some libraries for the JIT-compilation of
# programs for BrainScaleS-2's embedded processor.
ppu_include_dirs = []
- ppu_dep_names = ['bitsery', 'boost']
+ ppu_dep_names = ['bitsery', 'boost', 'cereal']
for ppu_dep_name in ppu_dep_names:
dep = self.spec[ppu_dep_name]
dep_include_dirs = set(dep.headers.directories)
--
GitLab