create new experimental release

.gitlab-ci.yml

@@ -7,6 +7,11 @@ variables:
   SPACK_VERSION: v0.21.1
   SPACK_PATH_GITLAB: /mnt/spack_v0.21.1
   SYSTEMNAME: ebrainslab
+  OC_PROJECT: jupyterhub
+
+# ===================================================================
+# LAB DEPLOYMENTS
+# ===================================================================
 
 # start an OpenShift Job that will build the Spack environment
 .deploy-build-environment:
@@ -43,170 +48,168 @@ variables:
   #     - kernel_specs
   #   when: always
 
-# Deploy in the lab-int environment the version of the tools to be
-# tested before released to production (push pipeline)
-# deploy on the dev environment of the okd dev cluster at CSCS
-# runs on protected branches only as the token variable is protected
-deploy-int-release-dev-cscs:
+# -------------------------------------------------------------------
+# Lab deployment target environments: dev and prod Lab instances
+# -------------------------------------------------------------------
+
+# deploy to a dev lab environment
+.deploy-dev-server:
   extends: .deploy-build-environment
   variables:
-    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
+    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
+    INSTALLATION_ROOT: /srv/test-build-2402
+
+# deploy to a prod lab environment
+.deploy-prod-server:
+  extends: .deploy-build-environment
+  variables:
+    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
+    INSTALLATION_ROOT: /srv/main-spack-instance-2402
+
+# deploy to the dev lab environment at CSCS
+.deploy-dev-server-cscs:
+  extends: .deploy-dev-server
+  variables:
     OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER
     OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN
+    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
     OC_PROJECT: jupyterhub-int
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
-    INSTALLATION_ROOT: /srv/test-build-2402
-    SPACK_ENV: test
-    RELEASE_NAME: EBRAINS-test
   resource_group: shared-NFS-mount-dev-cscs
   tags:             # this is just to ensure that the two jobs will run on different runners
     - read-write    # to avoid issues with common environment variables
     - shell-runner
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule"
 
-# Deploy in the lab-dev environment the version of the tools to be
-# tested before released to production (push pipeline)
-# deploy on the dev environment of the k8s dev cluster at CINECA
-# runs on protected branches only as the token variable is protected
-deploy-int-release-dev-cineca:
-  extends: .deploy-build-environment
-  variables:
-    OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER
-    OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
-    INSTALLATION_ROOT: /srv/test-build-2402
-    SPACK_ENV: test
-    RELEASE_NAME: EBRAINS-test
-  resource_group: shared-NFS-mount-dev-cineca
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule"
-
-# Deploy the production release of tools (manual pipeline)
-# deploy on the production environment of the okd prod cluster at CSCS
-# runs on protected branches only as the token variable is protected
-deploy-prod-release-prod-cscs:
-  extends: .deploy-build-environment
+# deploy to the prod lab environment at CSCS
+.deploy-prod-server-cscs:
+  extends: .deploy-prod-server
   variables:
-    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
     OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER
     OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
-    INSTALLATION_ROOT: /srv/main-spack-instance-2402
-    SPACK_ENV: ebrains-24-04
-    RELEASE_NAME: EBRAINS-24.04
+    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
   resource_group: shared-NFS-mount-prod-cscs
   tags:             # this is just to ensure that the two jobs will run on different runners
     - read-write    # to avoid issues with common environment variables
     - shell-runner
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^ebrains/
-      when: manual
 
-# Deploy the production release of tools (manual pipeline)
-# deploy on the prod environment of the k8s prod cluster at JSC
-# runs on protected branches only as the token variable is protected
-deploy-prod-release-prod-jsc:
-  extends: .deploy-build-environment
+# deploy to the dev lab environment at CINECA
+.deploy-dev-server-cineca:
+  extends: .deploy-dev-server
+  variables:
+    OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER
+    OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN
+  resource_group: shared-NFS-mount-dev-cineca
+  tags:             # this is just to ensure that the two jobs will run on different runners
+    - read-only     # to avoid issues with common environment variables
+    - shell-runner
+
+# deploy to the prod lab environment at JSC
+.deploy-prod-server-jsc:
+  extends: .deploy-prod-server
   variables:
     OPENSHIFT_SERVER: $JSC_K8S_PROD_SERVER
     OPENSHIFT_TOKEN: $JSC_K8S_PROD_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
-    INSTALLATION_ROOT: /srv/main-spack-instance-2402
-    SPACK_ENV: ebrains-24-04
-    RELEASE_NAME: EBRAINS-24.04
   resource_group: shared-NFS-mount-prod-jsc
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^ebrains/
-      when: manual
-
-# Deploy the experimental release of tools (sheduled pipeline)
-# once a week from latest working version of integration release 
-# (branch=experimental_release) to an experimental JupyterLab kernel
-# deploy on the dev environment of the okd dev cluster at CSCS
-# runs on protected branches only as the token variable is protected
-deploy-exp-release-dev-cscs:
-  extends: .deploy-build-environment
-  variables:
-    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
-    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER
-    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN
-    OC_PROJECT: jupyterhub-int
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
-    INSTALLATION_ROOT: /srv/test-build-2402
-    SPACK_ENV: experimental
-    RELEASE_NAME: EBRAINS-experimental
-  resource_group: shared-NFS-mount-dev-cscs
   tags:             # this is just to ensure that the two jobs will run on different runners
-    - read-write    # to avoid issues with common environment variables
+    - read-only     # to avoid issues with common environment variables
     - shell-runner
+
+# -------------------------------------------------------------------
+# Release types: test, experimental and official releases
+# -------------------------------------------------------------------
+
+# deploy int release (latest changes) to dev env to be tested before release to production
+.deploy-int-release:
+  variables:
+    SPACK_ENV: test
+    RELEASE_NAME: EBRAINS-test
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "dev"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule"
 
-# Deploy the experimental release of tools (sheduled pipeline)
-# once a week from latest working version of integration release
-# (branch=experimental_release) to an experimental JupyterLab kernel
-# deploy on the dev environment of the k8s dev cluster at CINECA
-# runs on protected branches only as the token variable is protected
-deploy-exp-release-dev-cineca:
-  extends: .deploy-build-environment
+# deploy the experimental release of tools once a week from latest working version of int release 
+.deploy-exp-release:
   variables:
-    OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER
-    OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int
-    INSTALLATION_ROOT: /srv/test-build-2402
     SPACK_ENV: experimental
     RELEASE_NAME: EBRAINS-experimental
-  resource_group: shared-NFS-mount-dev-cineca
+
+# deploy the experimental release to dev env
+.deploy-exp-dev-release:
+  extends: .deploy-exp-release
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "dev"
 
-# Deploy the experimental release of tools (sheduled pipeline)
-# once a week from latest working version of integration release 
-# (branch=experimental_release) to an experimental JupyterLab kernel
-# deploy on the prod environment of the okd prod cluster at CSCS
-# runs on protected branches only as the token variable is protected
-deploy-exp-release-prod-cscs:
-  extends: .deploy-build-environment
-  variables:
-    BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06
-    OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER
-    OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
-    INSTALLATION_ROOT: /srv/main-spack-instance-2402
-    SPACK_ENV: experimental
-    RELEASE_NAME: EBRAINS-experimental
-  resource_group: shared-NFS-mount-prod-cscs
-  tags:             # this is just to ensure that the two jobs will run on different runners
-    - read-write    # to avoid issues with common environment variables
-    - shell-runner
+# deploy the experimental release to prod env
+.deploy-exp-prod-release:
+  extends: .deploy-exp-release
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "prod"
 
-# Deploy the experimental release of tools (sheduled pipeline)
-# once a week from latest working version of integration release 
-# (branch=experimental_release) to an experimental JupyterLab kernel
-# deploy on the prod environment of the k8s prod cluster at JSC
-# runs on protected branches only as the token variable is protected
-deploy-exp-release-prod-jsc:
-  extends: .deploy-build-environment
+# deploy the production release of tools
+.deploy-prod-release:
   variables:
-    OPENSHIFT_SERVER: $JSC_K8S_PROD_SERVER
-    OPENSHIFT_TOKEN: $JSC_K8S_PROD_TOKEN
-    OC_PROJECT: jupyterhub
-    LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod
-    INSTALLATION_ROOT: /srv/main-spack-instance-2402
-    SPACK_ENV: experimental
-    RELEASE_NAME: EBRAINS-experimental
-  resource_group: shared-NFS-mount-prod-jsc
+    SPACK_ENV: ebrains-24-04
+    RELEASE_NAME: EBRAINS-24.04
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"  && $DEPLOYMENT == "prod"
+    - if: $CI_COMMIT_BRANCH =~ /^ebrains/
+      when: manual
+
+# -------------------------------------------------------------------
+# Lab deployment jobs
+# -------------------------------------------------------------------
+
+# deploy int release to dev environment at CSCS
+deploy-int-release-dev-cscs:
+  extends:
+    - .deploy-int-release
+    - .deploy-dev-server-cscs
+
+# deploy int release to dev environment at CINECA
+deploy-int-release-dev-cineca:
+  extends:
+    - .deploy-int-release
+    - .deploy-dev-server-cineca
+
+# deploy exp release to dev environment at CSCS
+deploy-exp-release-dev-cscs:
+  extends:
+    - .deploy-exp-dev-release
+    - .deploy-dev-server-cscs
+
+# deploy exp release to dev environment at CINECA
+deploy-exp-release-dev-cineca:
+  extends:
+    - .deploy-exp-dev-release
+    - .deploy-dev-server-cineca
+
+# deploy exp release to prod environment at CSCS
+deploy-exp-release-prod-cscs:
+  extends:
+    - .deploy-exp-prod-release
+    - .deploy-prod-server-cscs
 
+# deploy exp release to prod environment at JSC
+deploy-exp-release-prod-jsc:
+  extends:
+    - .deploy-exp-prod-release
+    - .deploy-prod-server-jsc
+
+# deploy prod release to prod environment at CSCS
+deploy-prod-release-prod-cscs:
+  extends:
+    - .deploy-prod-release
+    - .deploy-prod-server-cscs
+
+# deploy prod release to prod environment at JSC
+deploy-prod-release-prod-jsc:
+  extends:
+    - .deploy-prod-release
+    - .deploy-prod-server-jsc
+
+# ===================================================================
+# GITLAB RUNNER DEPLOYMENTS
+# ===================================================================
+
+# run test build job on any gitlab runner, trying to build latest changes
+# with persistent, read-only deployment as upstream
 build-spack-env-on-runner:
   stage: build
   tags:
@@ -243,6 +246,7 @@ build-spack-env-on-runner:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule" && $CI_PIPELINE_SOURCE != "merge_request_event"
 
+# update gitlab-runner upstream (read-only) installation
 sync-gitlab-spack-instance:
   stage: build
   tags:
@@ -279,6 +283,7 @@ sync-gitlab-spack-instance:
     - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "experimental_rel" || $CI_COMMIT_BRANCH =~ /^ebrains/) && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule"
       when: manual
 
+# run (scheduled) standalone tests for environment
 test-gitlab-spack-instance:
   stage: test
   tags:
@@ -303,4 +308,3 @@ test-gitlab-spack-instance:
     when: always
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" && $TEST_DEPLOYMENT == "true"
-