diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03448525a9d02699188380c30b6fce16781e21ad..502fc82fe7c2f7e5bba69a40d95a9fb87a28240c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,12 +172,13 @@ build-spack-env-on-runner: image: $BUILD_ENV_DOCKER_IMAGE variables: SPACK_DEV_ENV: ebrains-dev + SPACK_JOBS: 2 script: # deactivate environment views (we don't need them for the test build-job) - > echo " view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml # run installation script - - . install_spack_env.sh $CI_PROJECT_DIR $SPACK_VERSION $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB + - . install_spack_env.sh $SPACK_JOBS $CI_PROJECT_DIR $SPACK_VERSION $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB # re-activate envionment and run tests - spack env activate $SPACK_DEV_ENV # TODO: run all tests when test dependency issue is fixed @@ -207,6 +208,7 @@ sync-gitlab-spack-instance: image: $BUILD_ENV_DOCKER_IMAGE variables: SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds + SPACK_JOBS: 4 script: - SPACK_NFS_ENV=${CI_COMMIT_BRANCH//./-} # create spack dir if it doesn't exist @@ -214,7 +216,7 @@ sync-gitlab-spack-instance: # get latest state of EBRAINS repo - rm -rf $SPACK_REPO_PATH && cp -r $CI_PROJECT_DIR $SPACK_REPO_PATH # run installation script - - . install_spack_env.sh $SPACK_PATH_GITLAB $SPACK_VERSION $SPACK_REPO_PATH $SPACK_NFS_ENV + - . install_spack_env.sh $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_VERSION $SPACK_REPO_PATH $SPACK_NFS_ENV # create kernel spec, so that the environment can be used in gitlab CI jobs - RELEASE_NAME=$(case $CI_COMMIT_BRANCH in experimental_rel) echo ebrains-experimental;; ebrains*) echo ${CI_COMMIT_BRANCH:0:10}.${CI_COMMIT_BRANCH:11};; *) echo $CI_COMMIT_BRANCH;; esac); - . create_JupyterLab_kernel.sh $SPACK_PATH_GITLAB $SPACK_NFS_ENV $RELEASE_NAME /mnt/ebrains_env diff --git a/create_job.sh b/create_job.sh index a4083277e9c20d818343ef0a522072bdeda29469..19edcb63ddd35bc10cb393c78400550045fc8f76 100644 --- a/create_job.sh +++ b/create_job.sh @@ -61,7 +61,7 @@ spec: git clone https://gitlab.ebrains.eu/technical-coordination/project-internal/devops/platform/ebrains-spack-builds.git --branch \$BRANCH \$EBRAINS_REPO_PATH # run installation script - . \$EBRAINS_REPO_PATH/install_spack_env.sh \$INSTALLATION_ROOT \$SPACK_VERSION \$EBRAINS_REPO_PATH \$EBRAINS_SPACK_ENV + . \$EBRAINS_REPO_PATH/install_spack_env.sh \$SPACK_JOBS \$INSTALLATION_ROOT \$SPACK_VERSION \$EBRAINS_REPO_PATH \$EBRAINS_SPACK_ENV if [ \$? -eq 0 ] then @@ -93,6 +93,8 @@ spec: value: /srv/build_logs/$EBRAINS_SPACK_ENV - name: EBRAINS_REPO_PATH value: $INSTALLATION_ROOT/ebrains-spack-builds + - name: SPACK_JOBS + value: '4' volumes: - name: sharedbin persistentVolumeClaim: diff --git a/install_spack_env.sh b/install_spack_env.sh index 9762ac9b367f49b854d6fbef5712732f5b5180bc..1a120459d310672a9ffd9b62ee7a15f9f848fcb6 100644 --- a/install_spack_env.sh +++ b/install_spack_env.sh @@ -1,17 +1,18 @@ #!/bin/bash -# =============================================================================================================================== +# ========================================================================================================================================= # title : install_spack_env.sh -# usage : ./install_spack_env.sh $INSTALLATION_ROOT $SPACK_VERSION $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE +# usage : ./install_spack_env.sh $SPACK_JOBS $INSTALLATION_ROOT $SPACK_VERSION $EBRAINS_REPO $EBRAINS_SPACK_ENV $UPSTREAM_INSTANCE # description : installs or updates the spack environment defined in the EBRAINS spack repo # (if the specified spack instance doesn't exist, it also creates it) -# =============================================================================================================================== +# ========================================================================================================================================= -INSTALLATION_ROOT=$1 # where to set up the installation -SPACK_VERSION=$2 # which spack version to use -EBRAINS_REPO=$3 # location of ebrains-spack-builds repository -EBRAINS_SPACK_ENV=$4 # name of EBRAINS Spack environment to be created/updated -UPSTREAM_INSTANCE=$5 # path to Spack instance to use as upstream (optional) +SPACK_JOBS=$1 # number of jobs +INSTALLATION_ROOT=$2 # where to set up the installation +SPACK_VERSION=$3 # which spack version to use +EBRAINS_REPO=$4 # location of ebrains-spack-builds repository +EBRAINS_SPACK_ENV=$5 # name of EBRAINS Spack environment to be created/updated +UPSTREAM_INSTANCE=$6 # path to Spack instance to use as upstream (optional) SPACK_REPO=https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack.git SPACK_VERSION_EBRAINS=${SPACK_VERSION}_ebrains24.04 @@ -81,7 +82,7 @@ spack concretize --force --fresh --test root spack-python -c "exit(not len(spack.environment.active_environment().uninstalled_specs()))" && spack fetch --dependencies --missing # install the environment, use 2 jobs to reduce the amount of required RAM -spack install -y -j2 --fresh --test root +spack install -y -j$SPACK_JOBS --fresh --test root # rebuild spack's database spack reindex