Merge branch 'extend-install-script' into 'master'

use installation script in CI

See merge request technical-coordination/project-internal/devops/platform/ebrains-spack-builds!433

.gitlab-ci.yml

@@ -177,35 +177,13 @@ build-spack-env-on-runner:
   image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
   variables:
     SPACK_DEV_ENV: ebrains-dev
-    SPACK_DEV_PATH: $CI_PROJECT_DIR/spack
-    SPACK_USER_CACHE_PATH: $CI_PROJECT_DIR/.spack
-    SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack
   script:
-    - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH
-    # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
-    # quick fix for that: (TODO: investigate more and open PR)
-    - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_DEV_PATH/lib/spack/llnl/util/tty/log.py
-    - |
-      cat <<EOF > $SPACK_DEV_PATH/etc/spack/defaults/upstreams.yaml
-      upstreams:
-        ebrains-gitlab-spack-instance:
-          install_tree: $SPACK_PATH_GITLAB/spack/opt/spack
-      EOF
-    - . $SPACK_DEV_PATH/share/spack/setup-env.sh
-    - spack find
-    - spack load gcc@10.3.0
-    - spack compiler find
-    - spack repo add .
-    - spack env create $SPACK_DEV_ENV
-    - spack env activate $SPACK_DEV_ENV
-    - rm -rf $SPACK_ENV/site-config && cp -r site-config $SPACK_ENV
-    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
-    - cp /tmp/spack.yaml $SPACK_ENV/
-    - spack install -y -j2 --fresh --test root
+    # run installation script
+    - . install_spack_env.sh $CI_PROJECT_DIR $SPACK_VERSION $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB
   after_script:
-    - mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
+    - mkdir -p .R/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
       # for succesfully installed packages: keep the spack logs for any package modified during this CI job
-    - PKG_DIR=$SPACK_DEV_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
+    - PKG_DIR=$CI_PROJECT_DIR/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
     - if cd $PKG_DIR; then find . \( -name ".spack" -o -name ".build" -o -name ".spack_test_results" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/installed \;; fi
       # for not succesfully installed packages: also keep the spack logs for any packages that failed
     - if cd /tmp/$(whoami)/spack-stage/; then find . -maxdepth 2 -name "*.txt" -exec cp --parents "{}" $CI_PROJECT_DIR/spack_logs/not_installed \;; fi
@@ -217,12 +195,6 @@ build-spack-env-on-runner:
   except:
     variables:
       - $CI_PIPELINE_SOURCE == "schedule"
-  # cache:
-  #   key: spack-cache-$CI_COMMIT_REF_SLUG
-  #   paths:
-  #     - "$SPACK_DEV_PATH"
-  #     - "$SPACK_USER_CONFIG_PATH"
-  #   when: always
 
 sync-gitlab-spack-instance:
   stage: build
@@ -232,51 +204,17 @@ sync-gitlab-spack-instance:
   image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
   variables:
     SPACK_NFS_ENV: ebrains-runner-build
-    SPACK_PATH: $SPACK_PATH_GITLAB
-    SPACK_USER_CACHE_PATH: $SPACK_PATH/spack/.spack
-    SPACK_USER_CONFIG_PATH: $SPACK_PATH/spack/.spack
-    SPACK_REPO_PATH: $SPACK_PATH/ebrains-spack-builds
+    SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds
   script:
-    # clone spack if it doesn't exist
-    - |
-      if [ ! -d $SPACK_PATH/spack ]; then
-        git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_PATH/spack
-      fi
-    # SPACK PATCH: the post-build logs on install-time-test-logs.txt gets ovewritten by the post-install logs.
-    # quick fix for that: (TODO: investigate more and open PR)
-    - sed -i "s/self.file_like, \"w\"/self.file_like, \"a\"/g" $SPACK_PATH/spack/lib/spack/llnl/util/tty/log.py
-    # activate spack
-    - . $SPACK_PATH/spack/share/spack/setup-env.sh
     # get latest state of EBRAINS repo
-    - rm -rf $SPACK_REPO_PATH && mkdir $SPACK_REPO_PATH && cp -r -t $SPACK_REPO_PATH $CI_PROJECT_DIR/{packages,repo.yaml}
-    - spack repo list | grep -q ebrains-spack-builds && echo "Repository registered already" || spack repo add $SPACK_REPO_PATH
-    - spack repo list
-    # install compiler (if not already installed)
-    - EBRAINS_SPACK_COMPILER=$(grep 'compiler' $CI_PROJECT_DIR/site-config/$SYSTEMNAME/packages.yaml | awk -F'[][]' '{ print $2 }')
-    - spack compiler find
-    - spack compiler list | grep -q $EBRAINS_SPACK_COMPILER || spack install $EBRAINS_SPACK_COMPILER
-    - spack load $EBRAINS_SPACK_COMPILER
-    - spack compiler find
-    - spack compiler list
-    # create environment, if it doesn't exist
-    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV
-    # update environment site-configs
-    - rm -rf $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/site-config && cp -r site-config $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV
-    # update spack.yaml: merge top-level and site-specific spack.yaml files
-    - spack-python site-config/ymerge.py spack.yaml site-config/$SYSTEMNAME/spack.yaml > /tmp/spack.yaml
-    - cp /tmp/spack.yaml $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV
-    # There is a known spack bug (https://github.com/spack/spack/issues/29447) in installing test dependencies for installation tests. The workaround suggested
-    # in the issue is to NOT concretize separately, but simply remove the .lock file and let the enironment be concretized by the spack install command:
-    - rm $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.lock || echo "No spack.lock file"
-    # then activate environment
-    - spack env activate $SPACK_NFS_ENV
-    - spack install -y -j2 --fresh --test root
-    - spack reindex
+    - rm -rf $SPACK_REPO_PATH && cp -r $CI_PROJECT_DIR $SPACK_REPO_PATH
+    # run installation script
+    - . install_spack_env.sh $SPACK_PATH_GITLAB $SPACK_VERSION $SPACK_REPO_PATH $SPACK_NFS_ENV
   after_script:
     - mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
       # for succesfully installed packages: keep the spack logs for any package modified during this CI job
       # (we use repo.yaml, that is modified at each start of the pipeline, as a reference file)
-    - PKG_DIR=$SPACK_PATH/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
+    - PKG_DIR=$SPACK_PATH_GITLAB/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
     - if cd $PKG_DIR; then find . -newer $SPACK_REPO_PATH/repo.yaml \( -name ".spack" -o -name ".spack_test_results" -o -name ".build" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/installed \;; fi
       # for not succesfully installed packages: also keep the spack logs for any packages that failed
     - if cd /tmp/$(whoami)/spack-stage/; then find . -maxdepth 2 -name "*.txt" -exec cp --parents "{}" $CI_PROJECT_DIR/spack_logs/not_installed \;; fi

install_spack_env.sh

@@ -43,7 +43,7 @@ fi
 source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
 
 # add repo if it does not exist
-if [[ ! $(spack repo list | grep $EBRAINS_REPO) ]]
+if [[ ! $(spack repo list | grep ebrains-spack-builds$) ]]
 then
   spack repo add $EBRAINS_REPO
 fi