Merge branch 'master' into 'experimental_rel'

create new experimental release

See merge request technical-coordination/project-internal/devops/platform/ebrains-spack-builds!109

.gitlab-ci.yml

@@ -134,7 +134,8 @@ deploy-exp-release-dev-cscs:
          "argv": ["\$LAB_KERNEL_PATH/bin/env.sh", "{connection_file}", "--profile=default"],
          "display_name": "EBRAINS_experimental_release",
          "name": "spack_experimental_release",
-         "language": "python"
+         "language": "python",
+         "env": { "LAB_KERNEL_INFO": "spack_experimental_release_\$(date +"%d-%m-%y")" }
         }
         EOF
         EOS
@@ -175,7 +176,8 @@ deploy-exp-release-prod-cscs:
          "argv": ["\$LAB_KERNEL_PATH/bin/env.sh", "{connection_file}", "--profile=default"],
          "display_name": "EBRAINS_experimental_release",
          "name": "spack_experimental_release",
-         "language": "python"
+         "language": "python",
+         "env": { "LAB_KERNEL_INFO": "spack_experimental_release_\$(date +"%d-%m-%y")" }
         }
         EOF
         EOS
@@ -216,7 +218,8 @@ deploy-exp-release-prod-jsc:
          "argv": ["\$LAB_KERNEL_PATH/bin/env.sh", "{connection_file}", "--profile=default"],
          "display_name": "EBRAINS_experimental_release",
          "name": "spack_experimental_release",
-         "language": "python"
+         "language": "python",
+         "env": { "LAB_KERNEL_INFO": "spack_experimental_release_\$(date +"%d-%m-%y")" }
         }
         EOF
         EOS

operations/steps_to_update_experimental_release.txt

+Steps to update the current experimental release with no major upgrades (i.e. install it in the same Spack instance as before - respective CI/CD env variable is ${site}_INSTALLATION_ROOT_{env})
+
+1) Back up the Lab's experimental kernel configuration in both CSCS and JSC prod
+cp -r /srv/jupyterlab_kernels/prod/experimental /srv/backup/experimental_{date}
+2) Merge the master branch of the ebrains-spack-builds repo to the experimental_rel branch.
+Make sure that the master branch has uncommented in the spack.yaml file only those packages that have been built successfully.
+3) Start the scheduled pipeline with description "deploy experimental release in the dev CSCS cluster"
+4) Check if the pipeline executed successfully
+5) On Friday morning (not every Friday, could be when we have important or numerous changes) run the scheduled pipeline with description "deploy experimental release in prod environments".
+6) Only one job will continue (due to credentials mismatch in the gitlab runner)
+7) Check the result of the job. If it is successful retry the other job that failed previously. If it is not, restore the experimental kernel's configuration from step #1 in its original location.

packages/apbs/package.py

+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+
+class Apbs(CMakePackage):
+    """
+    APBS (Adaptive Poisson-Boltzmann Solver) solves the equations of continuum electrostatics
+    for large biomolecular assemblages. This software was designed “from the ground up”
+    using modern design principles to ensure its ability to interface with other computational
+    packages and evolve as methods and applications change over time. The APBS code is
+    accompanied by extensive documentation for both users and programmers and is supported
+    by a variety of utilities for preparing calculations and analyzing results.
+    Finally, the free, open-source APBS license ensures its accessibility to the entire
+    biomedical community.
+    """
+
+    # Homepage and Github URL.
+    homepage = "https://www.poissonboltzmann.org/"
+    url      = "https://github.com/Electrostatics/apbs/archive/refs/tags/v3.4.0.tar.gz"
+
+    # List of GitHub accounts to notify when the package is updated.
+    maintainers = ['thielblz', 'richtesn']
+
+    # SHA256 checksum.
+    version('3.4.0', sha256='572ff606974119430020ec948c78e171d8525fb0e67a56dad937a897cac67461')
+
+    # Dependencies.
+    depends_on('cmake@3.19', type='build')
+    depends_on('python@3.9:3.10', type=('build'))
+    depends_on('blas', type='build')
+    depends_on('suite-sparse', type='build')
+    depends_on('maloc', type='build')
+
+    def cmake_args(self):
+        # Min and max Python versions need to be set as variables to pass tests.
+        # See tests/CMakeLists.txt lines 6-14.
+        args = [
+                '-DPYTHON_MIN_VERSION=3.9',
+                '-DPYTHON_MAX_VERSION=3.10',
+                ]
+        return args

packages/arbor/package.py

+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Arbor(CMakePackage, CudaPackage):
+    """Arbor is a high-performance library for computational neuroscience
+    simulations."""
+
+    homepage = 'https://arbor-sim.org'
+    git      = 'https://github.com/arbor-sim/arbor.git'
+    url      = 'https://github.com/arbor-sim/arbor/releases/download/v0.6/arbor-v0.6-full.tar.gz'
+    maintainers = ['bcumming', 'brenthuisman', 'haampie', 'schmitts']
+
+    version('master', branch='master', submodules=True)
+    version('0.6', sha256='4cd333b18effc8833428ddc0b99e7dc976804771bc85da90034c272c7019e1e8', url='https://github.com/arbor-sim/arbor/releases/download/v0.6/arbor-v0.6-full.tar.gz')
+    version('0.5.2', sha256='290e2ad8ca8050db1791cabb6b431e7c0409c305af31b559e397e26b300a115d', url='https://github.com/arbor-sim/arbor/releases/download/v0.5.2/arbor-v0.5.2-full.tar.gz')
+    version('0.5', sha256='d0c8a4c7f97565d7c30493c66249be794d1dc424de266fc79cecbbf0e313df59', url='https://github.com/arbor-sim/arbor/releases/download/v0.5/arbor-v0.5-full.tar.gz')
+
+    variant('assertions', default=False, description='Enable arb_assert() assertions in code.')
+    variant('doc', default=False, description='Build documentation.')
+    variant('mpi', default=False, description='Enable MPI support')
+    variant('neuroml', default=True, description='Build NeuroML support library.')
+    variant('python', default=True, description='Enable Python frontend support')
+    variant('vectorize', default=False, description='Enable vectorization of computational kernels')
+
+    # https://docs.arbor-sim.org/en/latest/install/build_install.html?highlight=requirements#compilers
+    conflicts('%gcc@:8.3')
+    conflicts('%clang@:7')
+    # Cray compiler v9.2 and later is Clang-based.
+    conflicts('%cce@:9.1')
+    conflicts('%intel')
+
+    depends_on('cmake@3.12:', type='build')
+
+    # misc dependencies
+    depends_on('fmt@7.1:', when='@0.5.3:') # required by the modcc compiler
+    depends_on('nlohmann-json')
+    depends_on('cuda@10:', when='+cuda')
+    depends_on('libxml2', when='+neuroml')
+
+    # mpi
+    depends_on('mpi', when='+mpi')
+    depends_on('py-mpi4py', when='+mpi+python', type=('build', 'run'))
+
+    # python (bindings)
+    extends('python', when='+python')
+    depends_on('python@3.6:', when="+python", type=('build', 'run'))
+    depends_on('py-numpy', when='+python', type=('build', 'run'))
+    depends_on('py-pybind11@2.8.1:', when='+python', type=('build', 'run'))
+
+    # sphinx based documentation
+    depends_on('python@3.6:', when="+doc", type='build')
+    depends_on('py-sphinx', when="+doc", type='build')
+    depends_on('py-svgwrite', when='+doc', type='build')
+
+    @property
+    def build_targets(self):
+        return ['all', 'html'] if '+doc' in self.spec else ['all']
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant('ARB_WITH_ASSERTIONS', 'assertions'),
+            self.define_from_variant('ARB_WITH_MPI', 'mpi'),
+            self.define_from_variant('ARB_WITH_NEUROML', 'neuroml'),
+            self.define_from_variant('ARB_WITH_PYTHON', 'python'),
+            self.define_from_variant('ARB_VECTORIZE', 'vectorize'),
+        ]
+
+        if '+cuda' in self.spec:
+            args.append('-DARB_GPU=cuda')
+
+        # query spack for the architecture-specific compiler flags set by its wrapper
+        args.append('-DARB_ARCH=none')
+        opt_flags = self.spec.target.optimization_flags(
+            self.spec.compiler.name,
+            self.spec.compiler.version)
+        args.append('-DARB_CXX_FLAGS_TARGET=' + opt_flags)
+
+        return args

packages/oppulance/package.py

@@ -21,8 +21,8 @@ class Oppulance(Package):
 
     releases = [
         {
-            'version': '2.0-rc1',
-            'tag': 'ebrains_release-2-rc1'
+            'version': '2.0-rc5',
+            'tag': 'ebrains-2.0-rc5'
         },
     ]
 

packages/py-pdb2pqr/package.py

+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+
+class PyPdb2pqr(PythonPackage):
+    """
+    PDB2PQR - determining titration states,
+    adding missing atoms, and assigning
+    charges/radii to biomolecules.
+    """
+
+    # Url for the package's homepage.
+    homepage = "http://www.poissonboltzmann.org/"
+    pypi     = "pdb2pqr/pdb2pqr-3.5.2.tar.gz"
+
+    # List of GitHub accounts to
+    # notify when the package is updated.
+    maintainers = ['richtesn', 'thielblz']
+
+    version('3.5.2', sha256='9d145ff3797a563ce818f9d2488413ac339f66c58230670c2455b2572cccd957')
+    
+    depends_on('python@3.8:', type=('build','run'))

packages/py-pynn/package.py

@@ -23,6 +23,8 @@ class PyPynn(PythonPackage):
     version('0.8.1', sha256='ce94246284588414d1570c1d5d697805f781384e771816727c830b01ee30fe39')
     version('0.7.5', sha256='15f75f422f3b71c6129ecef23f29d8baeb3ed6502e7a321b8a2596c78ef7e03c')
 
+    variant('mpi', default=False, description='Enable MPI support')
+
     depends_on('python@2.6:2.8,3.3:')
     depends_on('py-jinja2@2.7:',        type=('build', 'run'))
     depends_on('py-docutils@0.10:',     type=('build', 'run'))
@@ -34,6 +36,9 @@ class PyPynn(PythonPackage):
     depends_on('py-neo@0.5.0:',         type=('build', 'run'), when="@0.9.0:")
     depends_on('py-lazyarray',          type=('build', 'run'), when="@0.9.0:")
 
+    depends_on('mpi', when='+mpi')
+    depends_on('py-mpi4py', type=('build', 'run'), when='+mpi')
+
     depends_on('py-mock@1.0:', type='test')
 
     patch('pynn-0.9.6-python3.patch', when='@0.9.6 ^python@3:')

packages/pynn-brainscales/package.py

@@ -9,20 +9,12 @@ class PynnBrainscales(WafPackage):
     """PyNN toplevel for the BrainScaleS-2 neuromorphic hardware systems"""
 
     homepage = "https://github.com/electronicvisions/pynn-brainscales"
-    #url      = "https://github.com/electronicvisions/pynn-brainscales/archive/v0.1.0.tar.gz"
     git      = "https://github.com/electronicvisions/pynn-brainscales.git"
 
-    version('2.0-rc1', branch='waf')
+    version('2.0-rc5', branch='waf')
 
     # PPU compiler dependencies
-    depends_on('gettext')
-    depends_on('zlib')
-    depends_on('bison')
-    depends_on('flex')
-    depends_on('m4')
-    depends_on('texinfo')
-    depends_on('wget')
-    conflicts('flex', when='@2.6.3', msg='Binutils 2.25 for Nux doesn\'t build with flex 2.6.3.')
+    depends_on('oppulance@2.0:')
 
     # host software dependencies
     depends_on('bitsery', type=('build', 'link', 'run'))
@@ -108,29 +100,35 @@ class PynnBrainscales(WafPackage):
     def configure(self, spec, prefix):
         """Setup and configure the project."""
 
-        self.waf('setup', '--repo-db-url=https://github.com/electronicvisions/projects', '--without-grenade-ppu-support', '--without-munge',
-            '--project=pynn-brainscales@ebrains_release-2-rc1',
-            '--project=haldls@ebrains_release-2-rc1',
-            '--project=grenade@ebrains_release-2-rc1',
-            '--project=code-format@ebrains_release-2-rc1',
-            '--project=logger@ebrains_release-2-rc1',
-            '--project=halco@ebrains_release-2-rc1',
-            '--project=hate@ebrains_release-2-rc1',
-            '--project=fisch@ebrains_release-2-rc1',
-            '--project=ztl@ebrains_release-2-rc1',
-            '--project=hxcomm@ebrains_release-2-rc1',
-            '--project=rant@ebrains_release-2-rc1',
-            '--project=pywrap@ebrains_release-2-rc1',
-            '--project=lib-boost-patches@ebrains_release-2-rc1',
-            '--project=sctrltp@ebrains_release-2-rc1',
-            '--project=hwdb@ebrains_release-2-rc1',
-            '--project=visions-slurm@ebrains_release-2-rc1',
-            '--project=flange@ebrains_release-2-rc1',
-            '--project=lib-rcf@ebrains_release-2-rc1',
-            '--project=bss-hw-params@ebrains_release-2-rc1'
+        self.waf('setup', '--repo-db-url=https://github.com/electronicvisions/projects', '--without-munge',
+            '--project=pynn-brainscales@ebrains-' + str(spec.version),
+            '--project=haldls@ebrains-' + str(spec.version),
+            '--project=grenade@ebrains-' + str(spec.version),
+            '--project=code-format@ebrains-' + str(spec.version),
+            '--project=logger@ebrains-' + str(spec.version),
+            '--project=halco@ebrains-' + str(spec.version),
+            '--project=hate@ebrains-' + str(spec.version),
+            '--project=fisch@ebrains-' + str(spec.version),
+            '--project=ztl@ebrains-' + str(spec.version),
+            '--project=hxcomm@ebrains-' + str(spec.version),
+            '--project=rant@ebrains-' + str(spec.version),
+            '--project=pywrap@ebrains-' + str(spec.version),
+            '--project=lib-boost-patches@ebrains-' + str(spec.version),
+            '--project=sctrltp@ebrains-' + str(spec.version),
+            '--project=hwdb@ebrains-' + str(spec.version),
+            '--project=visions-slurm@ebrains-' + str(spec.version),
+            '--project=flange@ebrains-' + str(spec.version),
+            '--project=lib-rcf@ebrains-' + str(spec.version),
+            '--project=bss-hw-params@ebrains-' + str(spec.version),
+            '--project=libnux@ebrains-' + str(spec.version)
          )
 
         args = ['--prefix={0}'.format(self.prefix)]
         args += self.configure_args()
 
         self.waf('configure', *args)
+
+    def build_args(self):
+        args = ['--keep', '--test-execnone', '-v']
+
+        return args

packages/sda/package.py

+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack import *
+
+
+class Sda(MakefilePackage):
+    """
+    SDA 7 can be used to carry out Brownian dynamics simulations of the diffusional association
+    in a continuum aqueous solvent of two solute molecules, e.g. proteins, or of a solute
+    molecule to an inorganic surface. SDA 7 can also be used to simulate the diffusion of
+    multiple proteins, in dilute or concentrated solutions, e.g., to study the effects of
+    macromolecular crowding.
+    """
+
+    # Url for the package's homepage.
+    homepage = "https://mcm.h-its.org/sda/doc/doc_sda7/doc/doc_sda7/index.html"
+    url      = "https://mcm.h-its.org/wiki/lib/exe/fetch.php?media=googlecolab:sda-7.3.3d.tgz"
+
+    # A list of GitHub accounts to notify when the package is updated.
+    maintainers = ['richtesn', 'thielblz']
+
+    # Versions and checksums.
+    version('7.3.3d', sha256='295ca0142532759a43375422a4ebc050e15b1c248e45493f5864765a9fe8076f')
+
+    # Dependencies.
+    depends_on('gcc@9.4.0:')
+    depends_on('python@3.8:', type=('build','run'))
+    conflicts('%intel')
+
+    build_directory = 'src'
+    build_jobs = 1
+
+    def edit(self, spec, prefix):
+        makefile = FileFilter('src/Makefile')
+        makefile.filter(r'gfortran', spack_fc)
+        makefile.filter(r'^\s*FC\s*=.*',  'FC = '  + spack_fc)
+        makefile.filter(r'^\s*CC_plus\s*=.*', 'CXX = ' + spack_cxx)
+        makefile.filter(r'\${CC_plus}', '${CXX}')
+        makefile.filter(r'^\s*CC_moins\s*=.*', 'CC = ' + spack_cc)
+        makefile.filter(r'\${CC_moins}', '${CC}')
+
+    @property
+    def install_targets(self):
+        build_directory = 'sda_flex-7.3.3d/src/'
+        targets = ['init libsda sda_flex tools auxi clust test_module']
+        return ['prefix={0}'.format(self.prefix), 'install']

spack.yaml

@@ -28,9 +28,8 @@ spack:
     - py-pyaescrypt
     - py-formencode
     - tvb-framework ^binutils+ld+gold
-    - pynn-brainscales@2.0-rc1 ^log4cxx@0.10.0 ^googletest@1.11.0:+gmock
+    - pynn-brainscales@2.0-rc5 ^log4cxx@0.10.0 ^googletest@1.11.0:+gmock
     - py-neo
-    #- oppulance@2.0-rc1
     #- py-cerebstats
     #- py-cerebunit
     - py-cerebrus
@@ -53,6 +52,9 @@ spack:
     - py-frites
     #- biobb-common
     #- biobb-md
+    #- apbs
+    - py-pdb2pqr
+    #- sda
     # demo for codejam12
     #- funniest1022
   concretization: together