From d73d2f4c876f4dc03733a40563344be7fe6474f3 Mon Sep 17 00:00:00 2001
From: Eleni Mathioulaki <emathioulaki@athenarc.gr>
Date: Fri, 21 Mar 2025 05:01:07 +0100
Subject: [PATCH] feat(CI): generate script that loads jupyter extensions
After each lab build job, generate a script that loads any jupyter extensions in the current env. The script will be used on Lab startup.
---
.gitlab-ci.yml | 11 ++++++
create_job_widget_script.sh | 68 +++++++++++++++++++++++++++++++++++++
2 files changed, 79 insertions(+)
create mode 100644 create_job_widget_script.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e8524909..e537adfe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,10 +4,12 @@ stages:
variables:
BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/ebrains-spack-build-env/base:devel
+ RUN_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/clb-jupyter-image/ebrains:dev-a7ab31be
SPACK_PATH_GITLAB: /mnt/spack_v0.23.1
SYSTEMNAME: ebrainslab
GIT_SUBMODULE_STRATEGY: recursive
GIT_CLEAN_FLAGS: -ffdxq
+ RUNNER_AFTER_SCRIPT_TIMEOUT: 20m
# ===================================================================
# LAB DEPLOYMENTS
@@ -47,6 +49,15 @@ variables:
- if [ $(kubectl get pods -l job-name=simplejob${CI_PIPELINE_ID} -o jsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}') -ne 0 ]; then exit 1; fi;
# delete the job, as we have the logs here
- kubectl delete job simplejob${CI_PIPELINE_ID} || true
+ after_script:
+ - kubectl config use-context $KUBE_CONTEXT
+ - sh create_job_widget_script.sh $CI_PIPELINE_ID $RUN_ENV_DOCKER_IMAGE $INSTALLATION_ROOT $SPACK_ENV $RELEASE_NAME $LAB_KERNEL_ROOT
+ - cat widget-script.yml
+ - kubectl create -f widget-script.yml
+ - while true; do sleep 300; x=$(kubectl get pods -l job-name=widget-script${CI_PIPELINE_ID} -o jsonpath='{.items[0].status.phase}'); if [ $x != "Running" ]; then break; fi; done
+ - kubectl logs jobs/widget-script${CI_PIPELINE_ID} | tee log.txt
+ - if [ $(kubectl get pods -l job-name=widget-script${CI_PIPELINE_ID} -o jsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}') -ne 0 ]; then exit 1; fi;
+ - kubectl delete job widget-script${CI_PIPELINE_ID} || true
# artifacts:
# paths:
# - spack_logs
diff --git a/create_job_widget_script.sh b/create_job_widget_script.sh
new file mode 100644
index 00000000..e28df2f8
--- /dev/null
+++ b/create_job_widget_script.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# ===========================================================================================================
+# title : create_job_widget_script.sh
+# usage : ./create_job_widget_script.sh $OC_JOB_ID $RUN_ENV_DOCKER_IMAGE $INSTALLATION_ROOT
+# $SPACK_ENV $RELEASE_NAME $LAB_KERNEL_ROOT
+# description : creates job file that generates a script to load the jupyter extensions for a given env
+# ===========================================================================================================
+
+OC_JOB_ID=$1
+RUN_ENV_DOCKER_IMAGE=$2
+INSTALLATION_ROOT=$3
+EBRAINS_SPACK_ENV=$4
+RELEASE_NAME=$5
+LAB_KERNEL_ROOT=$6
+
+cat <<EOT >> widget-script.yml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: widget-script${OC_JOB_ID}
+spec:
+ parallelism: 1
+ completions: 1
+ backoffLimit: 0
+ template:
+ spec:
+ containers:
+ - name: widget-script
+ image: ${RUN_ENV_DOCKER_IMAGE}
+ imagePullPolicy: Always
+ resources:
+ limits:
+ cpu: '1'
+ memory: '1Gi'
+ requests:
+ cpu: '0.5'
+ memory: '500Mi'
+ volumeMounts:
+ - name: sharedbin
+ mountPath: /srv
+ command:
+ - /bin/bash
+ - -c
+ - |
+ . \$INSTALLATION_ROOT/spack/share/spack/setup-env.sh
+ spack env activate --without-view \$EBRAINS_SPACK_ENV
+ KERNEL_PATH=\$LAB_KERNEL_ROOT/\$(echo "\$RELEASE_NAME" | tr '[:upper:]' '[:lower:]')
+ spack load --sh clb-nb-utils py-tvb-ext-bucket py-tvb-ext-unicore py-tvb-ext-xircuits > \$KERNEL_PATH/bin/widget_activation.sh
+ env:
+ - name: SYSTEMNAME
+ value: ebrainslab
+ - name: SPACK_DISABLE_LOCAL_CONFIG
+ value: "true"
+ - name: INSTALLATION_ROOT
+ value: "$INSTALLATION_ROOT"
+ - name: EBRAINS_SPACK_ENV
+ value: "$EBRAINS_SPACK_ENV"
+ - name: RELEASE_NAME
+ value: "$RELEASE_NAME"
+ - name: LAB_KERNEL_ROOT
+ value: "$LAB_KERNEL_ROOT"
+ volumes:
+ - name: sharedbin
+ persistentVolumeClaim:
+ claimName: shared-binaries
+ restartPolicy: Never
+EOT
--
GitLab