Merge branch 'master' into 'experimental_rel'

create new experimental release

See merge request technical-coordination/project-internal/devops/platform/ebrains-spack-builds!245

.gitlab-ci.yml

 stages:
   - build
-  - deploy
-  - sync
 
 variables:
   SPACK_VERSION: v0.18.1
@@ -9,7 +7,7 @@ variables:
 
 # start an OpenShift Job that will build the Spack environment
 .deploy-build-environment:
-  stage: deploy
+  stage: build
   script:
     # login and select project in openshift
     - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN"
@@ -162,8 +160,6 @@ deploy-exp-release-prod-jsc:
 
 build-spack-env-on-runner:
   stage: build
-  rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'master'
   tags:
     - docker-runner
     - read-only
@@ -173,10 +169,8 @@ build-spack-env-on-runner:
     SPACK_DEV_PATH: $CI_PROJECT_DIR/spack
     SPACK_USER_CACHE_PATH: $CI_PROJECT_DIR/.spack
     SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack
-    TMP: $CI_PROJECT_DIR/.spack-tmp
   script:
     - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH
-    - mkdir $TMP
     - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml
     - |
       cat <<EOF > $SPACK_DEV_PATH/etc/spack/defaults/upstreams.yaml
@@ -192,22 +186,25 @@ build-spack-env-on-runner:
     - spack env create $SPACK_DEV_ENV spack.yaml
     - spack env activate $SPACK_DEV_ENV
     - spack concretize -f --fresh
-    - spack install -y --fresh --no-check-signature
+    - spack install -y --fresh --no-check-signature --keep-stage
+  after_script:
+    - cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./
+    - mv ./tmp/$(whoami)/spack-stage spack-logs
+  timeout: 2 days
+  artifacts:
+    paths:
+      - spack-logs
+    when: always
   # cache:
   #   key: spack-cache-$CI_COMMIT_REF_SLUG
   #   paths:
   #     - "$SPACK_DEV_PATH"
   #     - "$SPACK_USER_CONFIG_PATH"
   #   when: always
-  timeout: 1 day
-  artifacts:
-    paths:
-      - .spack-tmp/root/spack-stage
-    when: on_failure
 
 
 sync-gitlab-spack-instance:
-  stage: sync
+  stage: build
   tags:
     - docker-runner
     - read-write
@@ -232,7 +229,7 @@ sync-gitlab-spack-instance:
     - spack load gcc@10.3.0
     - spack compiler find
     - spack compiler list
-    - spack install python@3.8.11 %gcc@10.3.0 || (cp -r /tmp/spack/spack-stage $CI_PROJECT_DIR/build_logs; exit 1)
+    - spack install --keep-stage python@3.8.11 %gcc@10.3.0
     # - for section in $(spack config list); do spack config blame $section; done
     - mkdir -p $SPACK_REPO_PATH && cp -r -t $SPACK_REPO_PATH $CI_PROJECT_DIR/{packages,repo.yaml}
     - spack repo list | grep -q ebrains-spack-builds && echo "Repository registered already" || spack repo add $SPACK_REPO_PATH
@@ -241,15 +238,17 @@ sync-gitlab-spack-instance:
     - spack env activate $SPACK_NFS_ENV
     - cp $CI_PROJECT_DIR/spack.yaml $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.yaml
     - spack concretize -f --fresh
-    - spack install -y --fresh --no-check-signature || (cp -r /tmp/spack/spack-stage $CI_PROJECT_DIR/build_logs; exit 1)
+    - spack install -y --fresh --no-check-signature --keep-stage
     - spack module tcl refresh -y
     - spack reindex
     - spack env loads -r
-  timeout: 2 days
+  after_script:
+    - cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./
+    - mv ./tmp/$(whoami)/spack-stage spack-logs
   artifacts:
     paths:
-      - build_logs/
-    when: on_failure
+      - spack-logs
+    when: always
   only:
     refs:
       - schedules

packages/hxtorch/package.py

@@ -2,9 +2,10 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+
 from spack import *
 from spack.pkg.builtin.boost import Boost
-import os
 
 
 class Hxtorch(WafPackage):
@@ -128,7 +129,7 @@ class Hxtorch(WafPackage):
         env.set('C_INCLUDE_PATH', ':'.join(include))
         env.set('CPLUS_INCLUDE_PATH', ':'.join(include))
         env.set('LIBRARY_PATH', ':'.join(library))
-        env.set('LD_LIBRARY_PATH', ':'.join(library))
+        env.set('WAF_CONFIGURE_LD_LIBRARY_PATH', ':'.join(library))
         env.prepend_path('PATH', ':'.join(path))
 
     def setup_run_environment(self, env):
@@ -164,6 +165,8 @@ class Hxtorch(WafPackage):
         args = ['--prefix={0}'.format(self.prefix)]
         args += self.configure_args()
 
+        env = os.environ
+        env['LD_LIBRARY_PATH'] = env.get('WAF_CONFIGURE_LD_LIBRARY_PATH')
         self.waf('configure', '--build-profile=release', *args)
 
     def build_args(self):

packages/nest/package.py

@@ -27,6 +27,7 @@ class Nest(CMakePackage):
     git      = "https://github.com/nest/nest-simulator.git"
 
     version('master', branch='master')
+    version('3.4_rc1',sha256='62558bf545abd5a54e92abd4ce8afaa58b45fe1065e6f47a7da723cd85a7e013')
     version('3.3',    sha256='179462b966cc61f5785d2fee770bc36f86745598ace9cd97dd620622b62043ed')
     version('3.2',    sha256='583d5725882ad5e8fd4fc7ffab425da97cbbb91fadbc327e940c184e8892b958')
     patch('nest-simulator-3.2-p1-VersionNumber.patch', when='@3.2')

packages/pynn-brainscales/package.py

@@ -2,6 +2,8 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+
 from spack import *
 from spack.pkg.builtin.boost import Boost
 
@@ -94,7 +96,7 @@ class PynnBrainscales(WafPackage):
         env.set('C_INCLUDE_PATH', ':'.join(include))
         env.set('CPLUS_INCLUDE_PATH', ':'.join(include))
         env.set('LIBRARY_PATH', ':'.join(library))
-        env.set('LD_LIBRARY_PATH', ':'.join(library))
+        env.set('WAF_CONFIGURE_LD_LIBRARY_PATH', ':'.join(library))
         env.prepend_path('PATH', ':'.join(path))
 
     def setup_run_environment(self, env):
@@ -130,6 +132,8 @@ class PynnBrainscales(WafPackage):
         args = ['--prefix={0}'.format(self.prefix)]
         args += self.configure_args()
 
+        env = os.environ
+        env['LD_LIBRARY_PATH'] = env.get('WAF_CONFIGURE_LD_LIBRARY_PATH')
         self.waf('configure', '--build-profile=release', *args)
 
     def build_args(self):

spack.yaml

@@ -19,7 +19,7 @@ spack:
     - py-lazyarray
     #- py-version-query
     # EBRAINS simulators
-    - nest@3.3 +python +gsl +mpi
+    - nest@3.4_rc1
     - arbor +python +mpi
     - neuron +mpi
     - py-pynn@0.10.1 +mpi