diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68f511b7f0f668dbdb870fe10df5a3601ec0a3ea..1ae570a4a9b5fc69358da568a755a557a4762b50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,11 @@ variables: SPACK_VERSION: v0.21.1 SPACK_PATH_GITLAB: /mnt/spack_v0.21.1 SYSTEMNAME: ebrainslab + OC_PROJECT: jupyterhub + +# =================================================================== +# LAB DEPLOYMENTS +# =================================================================== # start an OpenShift Job that will build the Spack environment .deploy-build-environment: @@ -43,170 +48,168 @@ variables: # - kernel_specs # when: always -# Deploy in the lab-int environment the version of the tools to be -# tested before released to production (push pipeline) -# deploy on the dev environment of the okd dev cluster at CSCS -# runs on protected branches only as the token variable is protected -deploy-int-release-dev-cscs: +# ------------------------------------------------------------------- +# Lab deployment target environments: dev and prod Lab instances +# ------------------------------------------------------------------- + +# deploy to a dev lab environment +.deploy-dev-server: extends: .deploy-build-environment variables: - BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 + LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int + INSTALLATION_ROOT: /srv/test-build-2402 + +# deploy to a prod lab environment +.deploy-prod-server: + extends: .deploy-build-environment + variables: + LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod + INSTALLATION_ROOT: /srv/main-spack-instance-2402 + +# deploy to the dev lab environment at CSCS +.deploy-dev-server-cscs: + extends: .deploy-dev-server + variables: OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN + BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 OC_PROJECT: jupyterhub-int - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int - INSTALLATION_ROOT: /srv/test-build-2402 - SPACK_ENV: test - RELEASE_NAME: EBRAINS-test resource_group: shared-NFS-mount-dev-cscs tags: # this is just to ensure that the two jobs will run on different runners - read-write # to avoid issues with common environment variables - shell-runner - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule" -# Deploy in the lab-dev environment the version of the tools to be -# tested before released to production (push pipeline) -# deploy on the dev environment of the k8s dev cluster at CINECA -# runs on protected branches only as the token variable is protected -deploy-int-release-dev-cineca: - extends: .deploy-build-environment - variables: - OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER - OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int - INSTALLATION_ROOT: /srv/test-build-2402 - SPACK_ENV: test - RELEASE_NAME: EBRAINS-test - resource_group: shared-NFS-mount-dev-cineca - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule" - -# Deploy the production release of tools (manual pipeline) -# deploy on the production environment of the okd prod cluster at CSCS -# runs on protected branches only as the token variable is protected -deploy-prod-release-prod-cscs: - extends: .deploy-build-environment +# deploy to the prod lab environment at CSCS +.deploy-prod-server-cscs: + extends: .deploy-prod-server variables: - BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod - INSTALLATION_ROOT: /srv/main-spack-instance-2402 - SPACK_ENV: ebrains-24-04 - RELEASE_NAME: EBRAINS-24.04 + BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 resource_group: shared-NFS-mount-prod-cscs tags: # this is just to ensure that the two jobs will run on different runners - read-write # to avoid issues with common environment variables - shell-runner - rules: - - if: $CI_COMMIT_BRANCH =~ /^ebrains/ - when: manual -# Deploy the production release of tools (manual pipeline) -# deploy on the prod environment of the k8s prod cluster at JSC -# runs on protected branches only as the token variable is protected -deploy-prod-release-prod-jsc: - extends: .deploy-build-environment +# deploy to the dev lab environment at CINECA +.deploy-dev-server-cineca: + extends: .deploy-dev-server + variables: + OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER + OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN + resource_group: shared-NFS-mount-dev-cineca + tags: # this is just to ensure that the two jobs will run on different runners + - read-only # to avoid issues with common environment variables + - shell-runner + +# deploy to the prod lab environment at JSC +.deploy-prod-server-jsc: + extends: .deploy-prod-server variables: OPENSHIFT_SERVER: $JSC_K8S_PROD_SERVER OPENSHIFT_TOKEN: $JSC_K8S_PROD_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod - INSTALLATION_ROOT: /srv/main-spack-instance-2402 - SPACK_ENV: ebrains-24-04 - RELEASE_NAME: EBRAINS-24.04 resource_group: shared-NFS-mount-prod-jsc - rules: - - if: $CI_COMMIT_BRANCH =~ /^ebrains/ - when: manual - -# Deploy the experimental release of tools (sheduled pipeline) -# once a week from latest working version of integration release -# (branch=experimental_release) to an experimental JupyterLab kernel -# deploy on the dev environment of the okd dev cluster at CSCS -# runs on protected branches only as the token variable is protected -deploy-exp-release-dev-cscs: - extends: .deploy-build-environment - variables: - BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 - OPENSHIFT_SERVER: $CSCS_OPENSHIFT_DEV_SERVER - OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_DEV_TOKEN - OC_PROJECT: jupyterhub-int - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int - INSTALLATION_ROOT: /srv/test-build-2402 - SPACK_ENV: experimental - RELEASE_NAME: EBRAINS-experimental - resource_group: shared-NFS-mount-dev-cscs tags: # this is just to ensure that the two jobs will run on different runners - - read-write # to avoid issues with common environment variables + - read-only # to avoid issues with common environment variables - shell-runner + +# ------------------------------------------------------------------- +# Release types: test, experimental and official releases +# ------------------------------------------------------------------- + +# deploy int release (latest changes) to dev env to be tested before release to production +.deploy-int-release: + variables: + SPACK_ENV: test + RELEASE_NAME: EBRAINS-test rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $DEPLOYMENT == "dev" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule" -# Deploy the experimental release of tools (sheduled pipeline) -# once a week from latest working version of integration release -# (branch=experimental_release) to an experimental JupyterLab kernel -# deploy on the dev environment of the k8s dev cluster at CINECA -# runs on protected branches only as the token variable is protected -deploy-exp-release-dev-cineca: - extends: .deploy-build-environment +# deploy the experimental release of tools once a week from latest working version of int release +.deploy-exp-release: variables: - OPENSHIFT_SERVER: $CINECA_K8S_DEV_SERVER - OPENSHIFT_TOKEN: $CINECA_K8S_DEV_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/int - INSTALLATION_ROOT: /srv/test-build-2402 SPACK_ENV: experimental RELEASE_NAME: EBRAINS-experimental - resource_group: shared-NFS-mount-dev-cineca + +# deploy the experimental release to dev env +.deploy-exp-dev-release: + extends: .deploy-exp-release rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $DEPLOYMENT == "dev" -# Deploy the experimental release of tools (sheduled pipeline) -# once a week from latest working version of integration release -# (branch=experimental_release) to an experimental JupyterLab kernel -# deploy on the prod environment of the okd prod cluster at CSCS -# runs on protected branches only as the token variable is protected -deploy-exp-release-prod-cscs: - extends: .deploy-build-environment - variables: - BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_23.06 - OPENSHIFT_SERVER: $CSCS_OPENSHIFT_PROD_SERVER - OPENSHIFT_TOKEN: $CSCS_OPENSHIFT_PROD_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod - INSTALLATION_ROOT: /srv/main-spack-instance-2402 - SPACK_ENV: experimental - RELEASE_NAME: EBRAINS-experimental - resource_group: shared-NFS-mount-prod-cscs - tags: # this is just to ensure that the two jobs will run on different runners - - read-write # to avoid issues with common environment variables - - shell-runner +# deploy the experimental release to prod env +.deploy-exp-prod-release: + extends: .deploy-exp-release rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $DEPLOYMENT == "prod" -# Deploy the experimental release of tools (sheduled pipeline) -# once a week from latest working version of integration release -# (branch=experimental_release) to an experimental JupyterLab kernel -# deploy on the prod environment of the k8s prod cluster at JSC -# runs on protected branches only as the token variable is protected -deploy-exp-release-prod-jsc: - extends: .deploy-build-environment +# deploy the production release of tools +.deploy-prod-release: variables: - OPENSHIFT_SERVER: $JSC_K8S_PROD_SERVER - OPENSHIFT_TOKEN: $JSC_K8S_PROD_TOKEN - OC_PROJECT: jupyterhub - LAB_KERNEL_ROOT: /srv/jupyterlab_kernels/prod - INSTALLATION_ROOT: /srv/main-spack-instance-2402 - SPACK_ENV: experimental - RELEASE_NAME: EBRAINS-experimental - resource_group: shared-NFS-mount-prod-jsc + SPACK_ENV: ebrains-24-04 + RELEASE_NAME: EBRAINS-24.04 rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $DEPLOYMENT == "prod" + - if: $CI_COMMIT_BRANCH =~ /^ebrains/ + when: manual + +# ------------------------------------------------------------------- +# Lab deployment jobs +# ------------------------------------------------------------------- + +# deploy int release to dev environment at CSCS +deploy-int-release-dev-cscs: + extends: + - .deploy-int-release + - .deploy-dev-server-cscs + +# deploy int release to dev environment at CINECA +deploy-int-release-dev-cineca: + extends: + - .deploy-int-release + - .deploy-dev-server-cineca + +# deploy exp release to dev environment at CSCS +deploy-exp-release-dev-cscs: + extends: + - .deploy-exp-dev-release + - .deploy-dev-server-cscs + +# deploy exp release to dev environment at CINECA +deploy-exp-release-dev-cineca: + extends: + - .deploy-exp-dev-release + - .deploy-dev-server-cineca + +# deploy exp release to prod environment at CSCS +deploy-exp-release-prod-cscs: + extends: + - .deploy-exp-prod-release + - .deploy-prod-server-cscs +# deploy exp release to prod environment at JSC +deploy-exp-release-prod-jsc: + extends: + - .deploy-exp-prod-release + - .deploy-prod-server-jsc + +# deploy prod release to prod environment at CSCS +deploy-prod-release-prod-cscs: + extends: + - .deploy-prod-release + - .deploy-prod-server-cscs + +# deploy prod release to prod environment at JSC +deploy-prod-release-prod-jsc: + extends: + - .deploy-prod-release + - .deploy-prod-server-jsc + +# =================================================================== +# GITLAB RUNNER DEPLOYMENTS +# =================================================================== + +# run test build job on any gitlab runner, trying to build latest changes +# with persistent, read-only deployment as upstream build-spack-env-on-runner: stage: build tags: @@ -243,6 +246,7 @@ build-spack-env-on-runner: rules: - if: $CI_PIPELINE_SOURCE != "schedule" && $CI_PIPELINE_SOURCE != "merge_request_event" +# update gitlab-runner upstream (read-only) installation sync-gitlab-spack-instance: stage: build tags: @@ -279,6 +283,7 @@ sync-gitlab-spack-instance: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "experimental_rel" || $CI_COMMIT_BRANCH =~ /^ebrains/) && $CI_PROJECT_NAMESPACE =~ /platform\/esd/ && $CI_PIPELINE_SOURCE != "schedule" when: manual +# run (scheduled) standalone tests for environment test-gitlab-spack-instance: stage: test tags: @@ -303,4 +308,3 @@ test-gitlab-spack-instance: when: always rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $TEST_DEPLOYMENT == "true" -