.gitlab-ci.yml

stages:
  - build

variables:
  SPACK_VERSION: v0.18.1
  BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_22.12

# start an OpenShift Job that will build the Spack environment
.deploy-build-environment:
  stage: build
  script:
    # login and select project in openshift
    - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN"
    - oc project $OC_PROJECT
    # create job description file
    - chmod a+x create_job.sh
    - ./create_job.sh $CI_PIPELINE_ID $BUILD_ENV_DOCKER_IMAGE update $INSTALLATION_ROOT $SPACK_VERSION $SPACK_ENV $CI_COMMIT_BRANCH $RELEASE_NAME $LAB_KERNEL_ROOT
    - cat simplejob.yml
    # start the deploy job
    - oc create -f simplejob.yml
    # wait for job to finish to get the logs
    - while true; do sleep 300; x=$(oc get pods | grep simplejob${CI_PIPELINE_ID} | awk '{ print $3}'); if [ $x != "Running" ]; then break; fi; done 
    - oc logs jobs/simplejob${CI_PIPELINE_ID} | tee log.txt
    # if spack install has failed, fail the pipeline
    - if [ $(cat log.txt | grep "No module available for package" | wc -l) -gt 0 ]; then exit 1; fi;
    # delete the job from OpenShift as we have the logs here
    - oc delete job simplejob${CI_PIPELINE_ID} || true
  tags:
    - shell-runner

# Deploy in the lab-int environment the version of the tools to be
# tested before released to production (push pipeline)
# deploy on the dev environment of the okd dev cluster at CSCS
# runs on protected branches only as the token variable is protected
deploy-int-release-dev-cscs:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER
    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN
    OC_PROJECT: jupyterhub-int
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
    INSTALLATION_ROOT: /srv/test-build-2212
    SPACK_ENV: test
    RELEASE_NAME: EBRAINS-test
  resource_group: shared-NFS-mount-dev-cscs
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /technical-coordination/

# Deploy the production release of tools (manual pipeline)
# deploy on the production environment of the okd prod cluster at CSCS
# runs on protected branches only as the token variable is protected
deploy-prod-release-prod-cscs:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER
    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN
    OC_PROJECT: jupyterhub
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
    INSTALLATION_ROOT: /srv/main-spack-instance-2212
    SPACK_ENV: ebrains-23-01
    RELEASE_NAME: EBRAINS-23.01
  resource_group: shared-NFS-mount-prod-cscs
  rules:
    - if: $CI_COMMIT_BRANCH =~ /ebrains/
      when: manual

# Deploy the production release of tools (manual pipeline)
# deploy on the production environment of the okd prod cluster at JSC
# runs on protected branches only as the token variable is protected
deploy-prod-release-prod-jsc:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $JSC_OPENSHIFT_PROD_SERVER
    OPENSHIFT_TOKEN: $JSC_OPENSHIFT_PROD_TOKEN
    OC_PROJECT: jupyterhub
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
    INSTALLATION_ROOT: /srv/main-spack-instance-2212
    SPACK_ENV: ebrains-23-01
    RELEASE_NAME: EBRAINS-23.01
  resource_group: shared-NFS-mount-prod-jsc
  rules:
    - if: $CI_COMMIT_BRANCH =~ /ebrains/
      when: manual

# Deploy the experimental release of tools (sheduled pipeline)
# once a week from latest working version of integration release 
# (branch=experimental_release) to an experimental JupyterLab kernel
# deploy on the dev environment of the okd dev cluster at CSCS
# runs on protected branches only as the token variable is protected
deploy-exp-release-dev-cscs:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER
    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN
    OC_PROJECT: jupyterhub-int
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
    INSTALLATION_ROOT: /srv/test-build-2212
    SPACK_ENV: experimental
    RELEASE_NAME: EBRAINS-experimental
  resource_group: shared-NFS-mount-dev-cscs
  only:
    refs:
      - schedules
    variables:
      - $DEPLOYMENT == "dev"

# Deploy the experimental release of tools (sheduled pipeline)
# once a week from latest working version of integration release 
# (branch=experimental_release) to an experimental JupyterLab kernel
# deploy on the prod environment of the okd prod cluster at CSCS
# runs on protected branches only as the token variable is protected
deploy-exp-release-prod-cscs:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER
    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN
    OC_PROJECT: jupyterhub
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
    INSTALLATION_ROOT: /srv/main-spack-instance-2212
    SPACK_ENV: experimental
    RELEASE_NAME: EBRAINS-experimental
  resource_group: shared-NFS-mount-prod-cscs
  tags:             # this is just to ensure that the two jobs will run on different runners
    - read-write    # to avoid issues with common environment variables
    - shell-runner
  only:
    refs:
      - schedules
    variables:
      - $DEPLOYMENT == "prod"

# Deploy the experimental release of tools (sheduled pipeline)
# once a week from latest working version of integration release 
# (branch=experimental_release) to an experimental JupyterLab kernel
# deploy on the prod environment of the okd prod cluster at JSC
# runs on protected branches only as the token variable is protected
deploy-exp-release-prod-jsc:
  extends: .deploy-build-environment
  variables:
    OPENSHIFT_SERVER: $JSC_OPENSHIFT_PROD_SERVER
    OPENSHIFT_TOKEN: $JSC_OPENSHIFT_PROD_TOKEN
    OC_PROJECT: jupyterhub
    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
    INSTALLATION_ROOT: /srv/main-spack-instance-2212
    SPACK_ENV: experimental
    RELEASE_NAME: EBRAINS-experimental
  resource_group: shared-NFS-mount-prod-jsc
  tags:             # this is just to ensure that the two jobs will run on different runners
    - read-only     # to avoid issues with common environment variables
    - shell-runner
  only:
    refs:
      - schedules
    variables:
      - $DEPLOYMENT == "prod"

build-spack-env-on-runner:
  stage: build
  tags:
    - docker-runner
    - read-only
  image: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners:gitlab_runners_22.09 
  variables:
    SPACK_DEV_ENV: ebrains-dev
    SPACK_DEV_PATH: $CI_PROJECT_DIR/spack
    SPACK_USER_CACHE_PATH: $CI_PROJECT_DIR/.spack
    SPACK_USER_CONFIG_PATH: $CI_PROJECT_DIR/.spack
  script:
    - git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_DEV_PATH
    - cp packages.yaml $SPACK_DEV_PATH/etc/spack/packages.yaml
    - |
      cat <<EOF > $SPACK_DEV_PATH/etc/spack/defaults/upstreams.yaml
      upstreams:
        ebrains-gitlab-spack-instance:
          install_tree: /mnt/spack_v0.18.1/opt/spack
      EOF
    - . $SPACK_DEV_PATH/share/spack/setup-env.sh
    - spack find
    - spack load gcc@10.3.0
    - spack compiler find
    - spack repo add .
    - spack env create $SPACK_DEV_ENV spack.yaml
    - spack env activate $SPACK_DEV_ENV
    - spack concretize -f --fresh
    - spack install -y -j2 --fresh --no-check-signature
  after_script:
    - mkdir spack_logs
    - |
      # succesfully installed packages: keep the spack logs for any package modified during this CI job
      SPACK_PACKAGES_DIR=$SPACK_DEV_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
      if cp --parents $SPACK_PACKAGES_DIR/*/.spack/*.txt ./; then
        mv .$SPACK_PACKAGES_DIR spack_logs/installed
      else
        echo "No packages installed, so no logs to collect"
      fi
      # not succesfully installed packages: also keep the spack logs for any packages that failed
      if cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./; then
        mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed
      else
        echo "No packages failed to build, so no logs to collect"
      fi
  artifacts:
    paths:
      - spack_logs
    when: always
  timeout: 2 days
  except:
    variables:
      - $CI_PIPELINE_SOURCE == "schedule"
  # cache:
  #   key: spack-cache-$CI_COMMIT_REF_SLUG
  #   paths:
  #     - "$SPACK_DEV_PATH"
  #     - "$SPACK_USER_CONFIG_PATH"
  #   when: always


sync-gitlab-spack-instance:
  stage: build
  tags:
    - docker-runner
    - read-write
  image: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:gitlab_runners_nfs_22.09 
  variables:
    SPACK_NFS_ENV: ebrains-runner-build
    SPACK_PATH: /mnt/spack_v0.18.1
    SPACK_USER_CACHE_PATH: $SPACK_PATH/.spack
    SPACK_USER_CONFIG_PATH: $SPACK_PATH/.spack
    SPACK_REPO_PATH: $SPACK_PATH/ebrains-spack-builds
  script:
    - |
      if [ ! -d $SPACK_PATH ]; then
        git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $SPACK_PATH
      fi
    - cp $CI_PROJECT_DIR/packages.yaml $SPACK_PATH/etc/spack/packages.yaml
    - . $SPACK_PATH/share/spack/setup-env.sh
    # - spack bootstrap untrust github-actions
    - spack compiler find
    - spack compiler list
    - spack load gcc@10.3.0 || spack install gcc@10.3.0
    - spack load gcc@10.3.0
    - spack compiler find
    - spack compiler list
    - spack install --keep-stage python@3.8.11 %gcc@10.3.0
    # - for section in $(spack config list); do spack config blame $section; done
    - rm -rf $SPACK_REPO_PATH && mkdir $SPACK_REPO_PATH && cp -r -t $SPACK_REPO_PATH $CI_PROJECT_DIR/{packages,repo.yaml}
    - spack repo list | grep -q ebrains-spack-builds && echo "Repository registered already" || spack repo add $SPACK_REPO_PATH
    - spack repo list
    - spack env list | grep -q $SPACK_NFS_ENV && echo "Environment created already" || spack env create $SPACK_NFS_ENV $CI_PROJECT_DIR/spack.yaml
    - spack env activate $SPACK_NFS_ENV
    - cp $CI_PROJECT_DIR/spack.yaml $SPACK_ROOT/var/spack/environments/$SPACK_NFS_ENV/spack.yaml
    - spack concretize -f --fresh
    - spack install -y -j2 --fresh --no-check-signature
    - spack module tcl refresh -y
    - spack reindex
    - spack env loads -r
  after_script:
    - mkdir spack_logs
    - |
      # succesfully installed packages: keep the spack logs for any package modified during this CI job
      # (we use repo.yaml, that is modified at each start of the pipeline, as a reference file)
      SPACK_PACKAGES_DIR=$SPACK_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
      if find $SPACK_PACKAGES_DIR/*/.spack/*.txt -newer $SPACK_REPO_PATH/repo.yaml -exec cp --parents -r "{}" ./ \;; then
        mv .$SPACK_PACKAGES_DIR spack_logs/installed
      else
        echo "No packages installed, so no logs to collect"
      fi
      # not succesfully installed packages: also keep the spack logs for any packages that failed
      if cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./; then
        mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed
      else
        echo "No packages failed to build, so no logs to collect"
      fi
  artifacts:
    paths:
      - spack_logs
    when: always
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /technical-coordination/
      when: manual