.gitlab-ci.yml

stages:
  - build
  - test

variables:
  BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/ebrains-spack-build-env/base:devel
  SPACK_PATH_GITLAB: /mnt/spack_v0.23.1
  SYSTEMNAME: ebrainslab
  GIT_SUBMODULE_STRATEGY: recursive
  GIT_CLEAN_FLAGS: -ffdxq

# ===================================================================
# LAB DEPLOYMENTS
# ===================================================================

# start a k8s Job that will build the Spack environment
.deploy-build-environment:
  variables:
    OCI_CACHE_PREFIX: ""
    UPDATE_SPACK_OCI_CACHES: false
  stage: build
  tags:
    - docker-runner
    - read-only
  image: alpine:3.21.0
  before_script:
    - apk add kubectl
  script:
    # use the site-specific kubectl context
    - kubectl config use-context $KUBE_CONTEXT
    # create job description file
    - sh create_job.sh $CI_PIPELINE_ID $BUILD_ENV_DOCKER_IMAGE $INSTALLATION_ROOT $SPACK_ENV $CI_COMMIT_SHA $RELEASE_NAME $LAB_KERNEL_ROOT $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
    - cat simplejob.yml
    # start the deploy job
    - kubectl create -f simplejob.yml
    # wait for job to finish to get the logs
    - while true; do sleep 300; x=$(kubectl get pods -l job-name=simplejob${CI_PIPELINE_ID} -o jsonpath='{.items[0].status.phase}'); if [ $x != "Running" ]; then break; fi; done
    # # copy logs of failed packages locally, to keep as job artifacts
    # - oc rsync $(oc get pods -l job-name=simplejob${CI_PIPELINE_ID} -o name):/tmp ./ --include="*/" --include="spack/spack-stage/*/*.txt" --exclude="*"
    # - mv tmp/spack/spack-stage spack_logs
    # # also copy the spec of the created kernel, to keep as job artifacts
    # - LAB_KERNEL_PATH = $LAB_KERNEL_ROOT/$(echo "$RELEASE_NAME" | tr '[:upper:]' '[:lower:]')
    # - oc rsync $(oc get pods -l job-name=simplejob${CI_PIPELINE_ID} -o name):$LAB_KERNEL_PATH ./
    # - mv .$LAB_KERNEL_PATH kernel_specs
    # if spack install has failed, fail the pipeline
    - kubectl logs jobs/simplejob${CI_PIPELINE_ID} | tee log.txt
    - if [ $(kubectl get pods -l job-name=simplejob${CI_PIPELINE_ID} -o jsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}') -ne 0 ]; then exit 1; fi;
    # delete the job, as we have the logs here
    - kubectl delete job simplejob${CI_PIPELINE_ID} || true
  # artifacts:
  #   paths:
  #     - spack_logs
  #     - kernel_specs
  #   when: always

# -------------------------------------------------------------------
# Lab deployment target environments: dev and prod Lab instances
# -------------------------------------------------------------------

# deploy to a dev lab environment
.deploy-dev-server:
  extends: .deploy-build-environment
  variables:
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
    INSTALLATION_ROOT: /srv/test-build-2502

# deploy to a prod lab environment
.deploy-prod-server:
  extends: .deploy-build-environment
  variables:
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
    INSTALLATION_ROOT: /srv/main-spack-instance-2502

# deploy to the dev lab environment at CINECA
.deploy-dev-server-cineca:
  extends: .deploy-dev-server
  variables:
    KUBE_CONTEXT: cineca-int
  resource_group: shared-NFS-mount-dev-cineca

# deploy to the prod lab environment at JSC
.deploy-prod-server-jsc:
  extends: .deploy-prod-server
  variables:
    KUBE_CONTEXT: jsc-prod
  resource_group: shared-NFS-mount-prod-jsc

# deploy to the prod lab environment at CINECA
.deploy-prod-server-cineca:
  extends: .deploy-prod-server
  variables:
    KUBE_CONTEXT: cineca-prod
  resource_group: shared-NFS-mount-prod-cineca

# -------------------------------------------------------------------
# Release types: test, experimental and official releases
# -------------------------------------------------------------------

# deploy int release (latest changes) to dev env to be tested before release to production
.deploy-int-release:
  variables:
    SPACK_ENV: test
    RELEASE_NAME: EBRAINS-test
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE == "push"

# deploy the experimental release of tools once a week from latest working version of int release 
.deploy-exp-release:
  variables:
    SPACK_ENV: experimental
    RELEASE_NAME: EBRAINS-experimental

# deploy the experimental release to dev env
.deploy-exp-dev-release:
  extends: .deploy-exp-release
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "dev"

# deploy the experimental release to prod env
.deploy-exp-prod-release:
  extends: .deploy-exp-release
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "prod"

# deploy the production release of tools
.deploy-prod-release:
  variables:
    SPACK_ENV: ebrains-24-04
    RELEASE_NAME: EBRAINS-24.04
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^ebrains/
      when: manual

# -------------------------------------------------------------------
# Lab deployment jobs
# -------------------------------------------------------------------

# deploy int release to dev environment at CINECA
deploy-int-release-dev-cineca:
  extends:
    - .deploy-int-release
    - .deploy-dev-server-cineca

# deploy exp release to dev environment at CINECA
deploy-exp-release-dev-cineca:
  extends:
    - .deploy-exp-dev-release
    - .deploy-dev-server-cineca

# deploy exp release to prod environment at JSC
deploy-exp-release-prod-jsc:
  extends:
    - .deploy-exp-prod-release
    - .deploy-prod-server-jsc

# deploy exp release to prod environment at CINECA
deploy-exp-release-prod-cineca:
  extends:
    - .deploy-exp-prod-release
    - .deploy-prod-server-cineca

# deploy prod release to prod environment at JSC
deploy-prod-release-prod-jsc:
  extends:
    - .deploy-prod-release
    - .deploy-prod-server-jsc

# deploy prod release to prod environment at CINECA
deploy-prod-release-prod-cineca:
  extends:
    - .deploy-prod-release
    - .deploy-prod-server-cineca

# ===================================================================
# GITLAB RUNNER DEPLOYMENTS
# ===================================================================

# run test build job on any gitlab runner, trying to build latest changes
# with persistent, read-only deployment as upstream
build-spack-env-on-runner:
  stage: build
  tags:
    - docker-runner
    - read-only
  image: $BUILD_ENV_DOCKER_IMAGE
  variables:
    SPACK_DEV_ENV: ebrains-dev
    SPACK_JOBS: 2
    OCI_CACHE_PREFIX: ""
    UPDATE_SPACK_OCI_CACHES: false
  script:
    # deactivate environment views (we don't need them for the test build-job)
    - >
        echo "  view: False" >> $CI_PROJECT_DIR/site-config/$SYSTEMNAME/spack.yaml
    # run installation script
    - bash install_spack_env.sh $SPACK_JOBS $CI_PROJECT_DIR $CI_PROJECT_DIR $SPACK_DEV_ENV $SPACK_PATH_GITLAB $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
  after_script:
    - mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
      # for succesfully installed packages: keep the spack logs for any package modified during this CI job
    - . $CI_PROJECT_DIR/spack/share/spack/setup-env.sh
    - cd $(spack-python -c "print(spack.store.parse_install_tree(spack.config.get('config'))[0])")
    - find . -mindepth 4 -maxdepth 4 \( -name ".spack" -o -name ".build" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/installed \;
      # for not succesfully installed packages: also keep the spack logs for any packages that failed
    - if cd /tmp/$(whoami)/spack-stage/; then find . -maxdepth 2 \( -name "*.txt" -o -name ".install_time_tests" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/not_installed \;; fi
    # - if [ -d /tmp/spack_tests ]; then mv /tmp/spack_tests $CI_PROJECT_DIR; fi
  artifacts:
    paths:
      - spack_logs
      # - spack_tests
    when: always
  timeout: 2 days
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"

# this one fills the spack caches and updates the ESD (ebrainslab-variant) images on harbor
sync-esd-image:
  stage: build
  tags:
    - esd_image
  image: docker-registry.ebrains.eu/esd/tmp:latest
  variables:
    CI_SPACK_ENV: esd
    SPACK_JOBS: 4
    # we access the branch-specific cache here (and also update it!)
    OCI_CACHE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}
    UPDATE_SPACK_OCI_CACHES: true
    OCI_IMAGE_PREFIX: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_COMMIT_BRANCH}/image
    INSTALLATION_ROOT: /esd
    SANDBOX_ROOT_RELATIVE: esd_image
    SANDBOX_ROOT: ${CI_PROJECT_DIR}/${SANDBOX_ROOT_RELATIVE}
  script:
    # create a sandbox/image for performing an spack install inside based on some base image/install
    - apptainer build --fix-perms --sandbox ${SANDBOX_ROOT}/ docker://${BUILD_ENV_DOCKER_IMAGE}
    # run installation script inside future container environment
    #   => DAG concretization, subsequent cache access + fetching and actual build should be separate steps
    - mkdir --mode=0777 -p ${SANDBOX_ROOT}/${INSTALLATION_ROOT}
    - apptainer exec --writable --bind ${CI_PROJECT_DIR}:${INSTALLATION_ROOT} --cwd ${INSTALLATION_ROOT} ${SANDBOX_ROOT} bash ./install_spack_env.sh $SPACK_JOBS $INSTALLATION_ROOT ${INSTALLATION_ROOT} $CI_SPACK_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
    - echo "export SYSTEMNAME=${SYSTEMNAME}" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
    - echo ". ${INSTALLATION_ROOT}/vendor/spack/var/spack/environments/${CI_SPACK_ENV}/load_env.sh" >> ${SANDBOX_ROOT}/.singularity.d/env/90-environment.sh
    # preparing to assemble the image: move in the CI project contents...
    - shopt -s dotglob
    - find . -maxdepth 1 -and -not -name "." -and -not -name "${SANDBOX_ROOT_RELATIVE}" -exec mv -t ${SANDBOX_ROOT}/${INSTALLATION_ROOT} {} \;
    # convert to SIF image file
    - apptainer build latest.sif "${SANDBOX_ROOT}"
    # upload SIF image as an artifact to OCI repo
    - oras push --username $HARBOR_USERNAME --password $HARBOR_PASSWORD ${OCI_IMAGE_PREFIX}:lab_latest.sif latest.sif && oras_ret=$? || oras_ret=$?
    # convert SIF image to OCI format and upload to OCI repo
    - skopeo --insecure-policy copy --dest-creds=${HARBOR_USERNAME}:${HARBOR_PASSWORD} sif:latest.sif docker://${OCI_IMAGE_PREFIX}:lab_latest.oci && skopeo_ret=$? || skopeo_ret=$?
    - exit $(expr $oras_ret + $skopeo_ret)
  after_script:
    - mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
      # for succesfully installed packages: keep the spack logs for any package modified during this CI job
    - PKG_DIR=${SANDBOX_ROOT}/${INSTALLATION_ROOT}/spack/opt/spack/**/linux-*/gcc-13.3.0
    - if cd $PKG_DIR; then find . \( -name ".spack" -o -name ".build" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/installed \;; fi
      # for not succesfully installed packages: also keep the spack logs for any packages that failed
    - if cd /tmp/$(whoami)/spack-stage/; then find . -maxdepth 2 \( -name "*.txt" -o -name ".install_time_tests" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/not_installed \;; fi
  artifacts:
    paths:
      - spack_logs
    when: always
  timeout: 2 days
  resource_group: registry-esd-master-image
  rules:
    # branches that update the gitlab-runner upstream (read-only) installation and the spack OCI caches
    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "experimental_rel" || $CI_COMMIT_BRANCH =~ /^ebrains/) && $CI_PROJECT_PATH =~ /platform\/esd\/ebrains-spack-builds/ && $CI_PIPELINE_SOURCE == "push"
      when: manual

# update gitlab-runner upstream (read-only) installation
sync-gitlab-spack-instance:
  stage: build
  tags:
    - docker-runner
    - read-write
  image: $BUILD_ENV_DOCKER_IMAGE
  variables:
    SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds
    SPACK_JOBS: 16
    OCI_CACHE_PREFIX: ""
    UPDATE_SPACK_OCI_CACHES: false
  script:
    - SPACK_NFS_ENV=${CI_COMMIT_BRANCH//./-}
    # create spack dir if it doesn't exist
    - mkdir -p $SPACK_PATH_GITLAB
    # get latest state of EBRAINS repo
    - if [ ! -d $SPACK_REPO_PATH ]; then git clone $CI_REPOSITORY_URL --recurse-submodules $SPACK_REPO_PATH; fi
    - cd $SPACK_REPO_PATH
    - git fetch origin
    - git reset --hard $CI_COMMIT_SHA
    - git submodule update --force
    # run installation script
    - bash install_spack_env.sh $SPACK_JOBS $SPACK_PATH_GITLAB $SPACK_REPO_PATH $SPACK_NFS_ENV "" $UPDATE_SPACK_OCI_CACHES $OCI_CACHE_PREFIX
    # create kernel spec, so that the environment can be used in gitlab CI jobs
    - RELEASE_NAME=$(case $CI_COMMIT_BRANCH in experimental_rel) echo ebrains-experimental;; ebrains*) echo ${CI_COMMIT_BRANCH:0:10}.${CI_COMMIT_BRANCH:11};; *) echo $CI_COMMIT_BRANCH;; esac);
    - bash create_JupyterLab_kernel.sh $SPACK_PATH_GITLAB $SPACK_NFS_ENV $RELEASE_NAME /mnt/ebrains_env
  after_script:
    - mkdir -p $CI_PROJECT_DIR/spack_logs/installed $CI_PROJECT_DIR/spack_logs/not_installed
      # for succesfully installed packages: keep the spack logs for any package modified during this CI job
      # (we use repo.yaml, that is modified at each start of the pipeline, as a reference file)
    - . $SPACK_PATH_GITLAB/spack/share/spack/setup-env.sh
    - cd $(spack-python -c "print(spack.store.parse_install_tree(spack.config.get('config'))[0])")
    - find . -mindepth 4 -maxdepth 4 -newer $SPACK_REPO_PATH/repo.yaml \( -name ".spack" -o -name ".build" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/installed \;
      # for not succesfully installed packages: also keep the spack logs for any packages that failed
    - if cd /tmp/$(whoami)/spack-stage/; then find . -maxdepth 2 \( -name "*.txt" -o -name ".install_time_tests" \) -exec cp -r --parents "{}" $CI_PROJECT_DIR/spack_logs/not_installed \;; fi
  artifacts:
    paths:
      - spack_logs
    when: always
  rules:
    # branches that update the gitlab-runner upstream (read-only) installation
    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "experimental_rel" || $CI_COMMIT_BRANCH =~ /^ebrains/) && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE == "push"
      when: manual

# run (scheduled) standalone tests for environment
test-gitlab-spack-instance:
  stage: test
  tags:
    - docker-runner
    - read-write
  image: $BUILD_ENV_DOCKER_IMAGE
  variables:
    SPACK_REPO_PATH: $SPACK_PATH_GITLAB/ebrains-spack-builds
    SPACK_USER_CACHE_PATH: $SPACK_PATH_GITLAB/spack/.spack
    SPACK_USER_CONFIG_PATH: $SPACK_PATH_GITLAB/spack/.spack
  script:
    - SPACK_NFS_ENV=${CI_COMMIT_BRANCH//./-}
    - source $SPACK_PATH_GITLAB/spack/share/spack/setup-env.sh
    - spack env activate $SPACK_NFS_ENV
    # TODO: run all tests when test dependency issue is fixed
    - spack test run -x wf-brainscales2-demos wf-multi-area-model
  after_script:
    - if [ -d /tmp/spack_tests ]; then mv /tmp/spack_tests $CI_PROJECT_DIR; fi
  artifacts:
    paths:
      - spack_tests
    when: always
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $TEST_DEPLOYMENT == "true"