Merge branch 'update-kernel-creation-script' into 'master'

Update kernel creation script

See merge request technical-coordination/project-internal/devops/platform/ebrains-spack-builds!268

create_JupyterLab_kernel.sh

@@ -12,67 +12,68 @@ EBRAINS_SPACK_ENV=$2
 RELEASE_NAME=$3
 LAB_KERNEL_ROOT=$4
 
-# enable modules
-source /usr/share/modules/init/sh
-
-# capture the empty env
-cd ~
-env >> /tmp/before.txt
-
-# load spack
-export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
-export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
-source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
-# no need to activate as the env is already activated in the context it is used
-# spack env activate $EBRAINS_SPACK_ENV
-
-# load modules
-module use $INSTALLATION_ROOT/spack/share/spack/modules/linux-ubuntu20.04-x86_64/
-source $INSTALLATION_ROOT/spack/var/spack/environments/$EBRAINS_SPACK_ENV/loads
-# also add user's .local python3.8 packages to allow package installation at runtime by the user using pip
-export PYTHONPATH=$PYTHONPATH:/opt/app-root/src/.local/lib/python3.8/site-packages
-export PATH=$PATH:/opt/app-root/src/.local/bin
-# and python modules installed in the base docker Collab image
-export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.8/dist-packages
-
-# capture the env after spack activation
-cd ~
-env >> /tmp/after.txt
-
 # kernel name is lowercase release name
 KERNEL_NAME=$(echo "$RELEASE_NAME" | tr '[:upper:]' '[:lower:]')
 # kernel dir is a directory inside LAB_KERNEL_ROOT named after the kernel
 KERNEL_PATH=$LAB_KERNEL_ROOT/$KERNEL_NAME
 
-# prepare the env file required for the JupyterLab kernel
+# prepare the base env.sh script required for the JupyterLab kernels:
 mkdir -p $KERNEL_PATH/bin
 
-# start of env creation
+# (1) create env.sh bash script
 cat <<EOF > $KERNEL_PATH/bin/env.sh
 #!/usr/bin/env bash
 set -euxo pipefail
 EOF
 
-# append the necessary env variables for spack env and tools
-diff /tmp/before.txt /tmp/after.txt|grep ">"|cut -c 3- |awk '$0="export "$0' >> $KERNEL_PATH/bin/env.sh
+# (2) append the necessary env variables for spack env and tools
+cat $INSTALLATION_ROOT/spack/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh >> $KERNEL_PATH/bin/env.sh
 
-# end of env creation
-cat <<EOF >>$KERNEL_PATH/bin/env.sh
-python -m ipykernel_launcher -f \$@
+# (3) also add the user's .local locations to allow package installation at runtime,
+# and the location of python modules installed in the base docker Collab image
+cat <<EOF >> $KERNEL_PATH/bin/env.sh
+export PATH=\$PATH:/opt/app-root/src/.local/bin
+export PYTHONPATH=\$PYTHONPATH:/opt/app-root/src/.local/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages
+export R_LIBS_USER=/opt/app-root/src/.local/lib/R/site-library
+mkdir -p \$R_LIBS_USER
+export R_LIBS=\$R_LIBS_USER:\$R_LIBS
 EOF
-chmod +x $KERNEL_PATH/bin/env.sh
 
-# create the new kernel's configuration file
+# (4) create startup script for python kernel
+cat $KERNEL_PATH/bin/env.sh > $KERNEL_PATH/bin/env_python.sh
+echo "python -m ipykernel_launcher -f \$@" >> $KERNEL_PATH/bin/env_python.sh
+chmod +x $KERNEL_PATH/bin/env_python.sh
+
+# ... and the new kernel's configuration file
 mkdir -p $KERNEL_PATH/$KERNEL_NAME
-cat <<EOF >$KERNEL_PATH/$KERNEL_NAME/kernel.json
+cat <<EOF > $KERNEL_PATH/$KERNEL_NAME/kernel.json
 {
- "argv": ["$KERNEL_PATH/bin/env.sh", "{connection_file}", "--profile=default"],
+ "argv": ["$KERNEL_PATH/bin/env_python.sh", "{connection_file}", "--profile=default"],
  "display_name": "$RELEASE_NAME",
- "name": "$KERNEL_NAME",
  "language": "python",
  "env": { "LAB_KERNEL_NAME": "$RELEASE_NAME", "LAB_KERNEL_RELEASE_DATE": "$(date +"%Y-%m-%d")" }
 }
 EOF
 
-# add EBRAINS logo to kernel
-cp $LAB_KERNEL_ROOT/../logo/logo-64x64.png $KERNEL_PATH/$KERNEL_NAME/
+# ... and add EBRAINS logo to kernel
+cp $LAB_KERNEL_ROOT/../etc/logo-64x64.png $KERNEL_PATH/$KERNEL_NAME/
+
+# (5) create startup script for R kernel
+cat $KERNEL_PATH/bin/env.sh > $KERNEL_PATH/bin/env_r.sh
+echo "R --slave -e \"IRkernel::main()\" --args \$@" >> $KERNEL_PATH/bin/env_r.sh
+chmod +x $KERNEL_PATH/bin/env_r.sh
+
+# ... and the new R kernel's configuration file
+mkdir -p $KERNEL_PATH/${KERNEL_NAME}-R
+cat <<EOF > $KERNEL_PATH/${KERNEL_NAME}-R/kernel.json
+{
+ "argv": ["$KERNEL_PATH/bin/env_r.sh", "{connection_file}"],
+ "display_name": "R-$RELEASE_NAME",
+ "language": "R",
+ "env": { "LAB_KERNEL_NAME": "R-$RELEASE_NAME", "LAB_KERNEL_RELEASE_DATE": "$(date +"%Y-%m-%d")" }
+}
+EOF
+
+# ... and add EBRAINS logo and kernel.js to kernel
+cp $LAB_KERNEL_ROOT/../etc/logo-64x64.png $KERNEL_PATH/${KERNEL_NAME}-R/
+cp $LAB_KERNEL_ROOT/../etc/kernel.js $KERNEL_PATH/${KERNEL_NAME}-R/

install_spack_env.sh

@@ -16,17 +16,17 @@ EBRAINS_SPACK_ENV=$4
 export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
 export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
 
-# initial setup: if spack dir doesn't already exist, clone it and install gcc and python
+# initial setup: clone spack if spack dir doesn't already exist and activate
 if [ ! -d $INSTALLATION_ROOT/spack ]
 then
-  # clone spack repo, import packages.yaml config file and activate
   git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $INSTALLATION_ROOT/spack
 fi
 cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml
 source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
+
 # install platform compiler and python (extract versions from packages.yaml)
 EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }')
-EBRAINS_SPACK_PYTHON=python@$(grep -A1 'python' $EBRAINS_REPO/packages.yaml | tail -n1 | awk -F'[][]' '{ print $2 }')
+EBRAINS_SPACK_PYTHON=python@$(grep -A1 'python:$' $EBRAINS_REPO/packages.yaml | tail -n1 | awk -F'[][]' '{ print $2 }')
 spack compiler find
 spack install $EBRAINS_SPACK_COMPILER
 spack load $EBRAINS_SPACK_COMPILER
@@ -52,10 +52,16 @@ spack env activate $EBRAINS_SPACK_ENV
 spack concretize --fresh --force
 # install the environment, use 2 jobs to reduce the amount of required RAM
 spack install --fresh --no-check-signature -y -j2
-
-# create modules files with spack
-spack module tcl refresh -y
 # rebuild spack's database
 spack reindex
-# create loads script that when sourced activates and loads the installed spack environment
-spack env loads -r
+
+# create load script that when sourced activates and loads the installed spack environment, using views
+# this needs deactivating the environment first:
+spack env deactivate
+unset SPACK_LD_LIBRARY_PATH
+spack env activate --sh $EBRAINS_SPACK_ENV > $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh
+
+# create modules files with spack
+# spack module tcl refresh -y
+# create loads script that when sourced activates and loads the installed spack environment, using modules
+# spack env loads -r

packages.yaml

@@ -4,6 +4,9 @@ packages:
         compiler: [gcc@10.3.0]
     python:
         version: [3.8.11]
+    r:
+        version: [4.1.3]
+        variants: [+X]
     py-torch:
         version: [1.11.0]
         variants: [~cuda~rocm~valgrind~mkldnn~mpi~gloo+tensorpipe~onnx_ml]

spack.yaml

 spack:
   specs:
+    - r-irkernel
     # Notebook
     #- py-jupyter
     - py-ipython