feat: configurable # of jobs in build script

.gitlab-ci.yml

@@ -172,12 +172,13 @@ build-spack-env-on-runner:
   image: $BUILD_ENV_DOCKER_IMAGE
   variables:
     SPACK_DEV_ENV: ebrains-dev
+    SPACK_JOBS: 2
   script:
     # deactivate environment views (we don't need them for the test build-job)
     - >
         echo "  view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
     # run installation script
-    - . install_spack_env.sh $CI_PROJECT_DIR $SPACK_VERSION $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB
+    - . install_spack_env.sh $SPACK_JOBS $CI_PROJECT_DIR $SPACK_VERSION $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB
     # re-activate envionment and run tests
     - spack env activate $SPACK_DEV_ENV
     # TODO: run all tests when test dependency issue is fixed
@@ -207,6 +208,7 @@ sync-gitlab-spack-instance:
   image: $BUILD_ENV_DOCKER_IMAGE
   variables:
     SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds
+    SPACK_JOBS: 4
   script:
     - SPACK_NFS_ENV=${CI_COMMIT_BRANCH//./-}
     # create spack dir if it doesn't exist
@@ -214,7 +216,7 @@ sync-gitlab-spack-instance:
     # get latest state of EBRAINS repo
     - rm -rf $SPACK_REPO_PATH && cp -r $CI_PROJECT_DIR $SPACK_REPO_PATH
     # run installation script
-    - . install_spack_env.sh $SPACK_PATH_GITLAB $SPACK_VERSION $SPACK_REPO_PATH $SPACK_NFS_ENV
+    - . install_spack_env.sh $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_VERSION $SPACK_REPO_PATH $SPACK_NFS_ENV
     # create kernel spec, so that the environment can be used in gitlab CI jobs
     - RELEASE_NAME=$(case $CI_COMMIT_BRANCH in experimental_rel) echo ebrains-experimental;; ebrains*) echo ${CI_COMMIT_BRANCH:0:10}.${CI_COMMIT_BRANCH:11};; *) echo $CI_COMMIT_BRANCH;; esac);
     - . create_JupyterLab_kernel.sh $SPACK_PATH_GITLAB $SPACK_NFS_ENV $RELEASE_NAME /mnt/ebrains_env

create_job.sh

@@ -61,7 +61,7 @@ spec:
           git clone https://gitlab.ebrains.eu/technical-coordination/project-internal/devops/platform/ebrains-spack-builds.git --branch \$BRANCH \$EBRAINS_REPO_PATH
           
           # run installation script
-          . \$EBRAINS_REPO_PATH/install_spack_env.sh \$INSTALLATION_ROOT \$SPACK_VERSION \$EBRAINS_REPO_PATH \$EBRAINS_SPACK_ENV
+          . \$EBRAINS_REPO_PATH/install_spack_env.sh \$SPACK_JOBS \$INSTALLATION_ROOT \$SPACK_VERSION \$EBRAINS_REPO_PATH \$EBRAINS_SPACK_ENV
           
           if [ \$? -eq 0 ]
           then
@@ -93,6 +93,8 @@ spec:
             value: /srv/build_logs/$EBRAINS_SPACK_ENV
           - name: EBRAINS_REPO_PATH
             value: $INSTALLATION_ROOT/ebrains-spack-builds
+          - name: SPACK_JOBS
+            value: '4'
       volumes:
         - name: sharedbin
           persistentVolumeClaim:

install_spack_env.sh

 #!/bin/bash
 
-# ===============================================================================================================================
+# =========================================================================================================================================
 # title         : install_spack_env.sh
-# usage         : ./install_spack_env.sh $INSTALLATION_ROOT $SPACK_VERSION $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE
+# usage         : ./install_spack_env.sh $SPACK_JOBS $INSTALLATION_ROOT $SPACK_VERSION $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE
 # description   : installs or updates the spack environment defined in the EBRAINS spack repo
 #                 (if the specified spack instance doesn't exist, it also creates it)
-# ===============================================================================================================================
+# =========================================================================================================================================
 
-INSTALLATION_ROOT=$1  # where to set up the installation
-SPACK_VERSION=$2      # which spack version to use
-EBRAINS_REPO=$3       # location of ebrains-spack-builds repository
-EBRAINS_SPACK_ENV=$4  # name of EBRAINS Spack environment to be created/updated
-UPSTREAM_INSTANCE=$5  # path to Spack instance to use as upstream (optional)
+SPACK_JOBS=$1         # number of jobs
+INSTALLATION_ROOT=$2  # where to set up the installation
+SPACK_VERSION=$3      # which spack version to use
+EBRAINS_REPO=$4       # location of ebrains-spack-builds repository
+EBRAINS_SPACK_ENV=$5  # name of EBRAINS Spack environment to be created/updated
+UPSTREAM_INSTANCE=$6  # path to Spack instance to use as upstream (optional)
 
 SPACK_REPO=https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack.git
 SPACK_VERSION_EBRAINS=${SPACK_VERSION}_ebrains24.04
@@ -81,7 +82,7 @@ spack concretize --force --fresh --test root
 spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && spack fetch --dependencies --missing
 
 # install the environment, use 2 jobs to reduce the amount of required RAM
-spack install -y -j2 --fresh --test root
+spack install -y -j$SPACK_JOBS --fresh --test root
 
 # rebuild spack's database
 spack reindex