Merge branch 'master' into 'add-r-sbtabvfgen'

.gitlab-ci.yml

@@ -3,7 +3,7 @@ stages:
 
 variables:
   SPACK_VERSION: v0.18.1
-  BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_22.12 
+  BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/okd:okd_22.12
 
 # start an OpenShift Job that will build the Spack environment
 .deploy-build-environment:
@@ -43,8 +43,8 @@ deploy-int-release-dev-cscs:
     SPACK_ENV: test
     RELEASE_NAME: EBRAINS-test
   resource_group: shared-NFS-mount-dev-cscs
-  only:
-    - master
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /technical-coordination/
 
 # Deploy the production release of tools (manual pipeline)
 # deploy on the production environment of the okd prod cluster at CSCS
@@ -61,7 +61,7 @@ deploy-prod-release-prod-cscs:
     RELEASE_NAME: EBRAINS-23.01
   resource_group: shared-NFS-mount-prod-cscs
   rules:
-    - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH =~ /ebrains/'
+    - if: $CI_COMMIT_BRANCH =~ /ebrains/
       when: manual
 
 # Deploy the production release of tools (manual pipeline)
@@ -79,7 +79,7 @@ deploy-prod-release-prod-jsc:
     RELEASE_NAME: EBRAINS-23.01
   resource_group: shared-NFS-mount-prod-jsc
   rules:
-    - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH =~ /ebrains/'
+    - if: $CI_COMMIT_BRANCH =~ /ebrains/
       when: manual
 
 # Deploy the experimental release of tools (sheduled pipeline)
@@ -185,12 +185,20 @@ build-spack-env-on-runner:
     - spack install -y -j2 --fresh --no-check-signature
   after_script:
     - mkdir spack_logs
-    # succesfully installed packages: keep the spack logs for any package modified during this CI job
-    - cp --parents $SPACK_DEV_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0/*/.spack/*.txt ./
-    - mv .$SPACK_DEV_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0 spack_logs/installed || echo "No logs for installed packages"
-    # not succesfully installed packages: also keep the spack logs for any packages that failed
-    - cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./
-    - mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed || echo "No logs for not-installed packages"
+    - |
+      # succesfully installed packages: keep the spack logs for any package modified during this CI job
+      SPACK_PACKAGES_DIR=$SPACK_DEV_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
+      if cp --parents $SPACK_PACKAGES_DIR/*/.spack/*.txt ./; then
+        mv .$SPACK_PACKAGES_DIR spack_logs/installed
+      else
+        echo "No packages installed, so no logs to collect"
+      fi
+      # not succesfully installed packages: also keep the spack logs for any packages that failed
+      if cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./; then
+        mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed
+      else
+        echo "No packages failed to build, so no logs to collect"
+      fi
   artifacts:
     paths:
       - spack_logs
@@ -248,19 +256,25 @@ sync-gitlab-spack-instance:
     - spack env loads -r
   after_script:
     - mkdir spack_logs
-    # succesfully installed packages: keep the spack logs for any package modified during this CI job
-    # (we use repo.yaml, that is modified at each start of the pipeline, as a reference file)
-    - find $SPACK_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0/*/.spack/*.txt -newer $SPACK_REPO_PATH/repo.yaml -exec cp --parents -r "{}" ./ \;
-    - mv .$SPACK_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0 spack_logs/installed || echo "No logs for installed packages"
-    # not succesfully installed packages: also keep the spack logs for any packages that failed
-    - cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./
-    - mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed || echo "No logs for not-installed packages"
+    - |
+      # succesfully installed packages: keep the spack logs for any package modified during this CI job
+      # (we use repo.yaml, that is modified at each start of the pipeline, as a reference file)
+      SPACK_PACKAGES_DIR=$SPACK_PATH/opt/spack/linux-ubuntu20.04-x86_64/gcc-10.3.0
+      if find $SPACK_PACKAGES_DIR/*/.spack/*.txt -newer $SPACK_REPO_PATH/repo.yaml -exec cp --parents -r "{}" ./ \;; then
+        mv .$SPACK_PACKAGES_DIR spack_logs/installed
+      else
+        echo "No packages installed, so no logs to collect"
+      fi
+      # not succesfully installed packages: also keep the spack logs for any packages that failed
+      if cp --parents /tmp/$(whoami)/spack-stage/*/*.txt ./; then
+        mv ./tmp/$(whoami)/spack-stage spack_logs/not_installed
+      else
+        echo "No packages failed to build, so no logs to collect"
+      fi
   artifacts:
     paths:
       - spack_logs
     when: always
-  only:
-    refs:
-      - schedules
-    variables:
-      - $DEPLOYMENT == "gitlab-nfs"
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE =~ /technical-coordination/
+      when: manual

create_JupyterLab_kernel.sh

@@ -12,67 +12,68 @@ EBRAINS_SPACK_ENV=$2
 RELEASE_NAME=$3
 LAB_KERNEL_ROOT=$4
 
-# enable modules
-source /usr/share/modules/init/sh
-
-# capture the empty env
-cd ~
-env >> /tmp/before.txt
-
-# load spack
-export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
-export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
-source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
-# no need to activate as the env is already activated in the context it is used
-# spack env activate $EBRAINS_SPACK_ENV
-
-# load modules
-module use $INSTALLATION_ROOT/spack/share/spack/modules/linux-ubuntu20.04-x86_64/
-source $INSTALLATION_ROOT/spack/var/spack/environments/$EBRAINS_SPACK_ENV/loads
-# also add user's .local python3.8 packages to allow package installation at runtime by the user using pip
-export PYTHONPATH=$PYTHONPATH:/opt/app-root/src/.local/lib/python3.8/site-packages
-export PATH=$PATH:/opt/app-root/src/.local/bin
-# and python modules installed in the base docker Collab image
-export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.8/dist-packages
-
-# capture the env after spack activation
-cd ~
-env >> /tmp/after.txt
-
 # kernel name is lowercase release name
 KERNEL_NAME=$(echo "$RELEASE_NAME" | tr '[:upper:]' '[:lower:]')
 # kernel dir is a directory inside LAB_KERNEL_ROOT named after the kernel
 KERNEL_PATH=$LAB_KERNEL_ROOT/$KERNEL_NAME
 
-# prepare the env file required for the JupyterLab kernel
+# prepare the base env.sh script required for the JupyterLab kernels:
 mkdir -p $KERNEL_PATH/bin
 
-# start of env creation
+# (1) create env.sh bash script
 cat <<EOF > $KERNEL_PATH/bin/env.sh
 #!/usr/bin/env bash
 set -euxo pipefail
 EOF
 
-# append the necessary env variables for spack env and tools
-diff /tmp/before.txt /tmp/after.txt|grep ">"|cut -c 3- |awk '$0="export "$0' >> $KERNEL_PATH/bin/env.sh
+# (2) append the necessary env variables for spack env and tools
+cat $INSTALLATION_ROOT/spack/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh >> $KERNEL_PATH/bin/env.sh
 
-# end of env creation
-cat <<EOF >>$KERNEL_PATH/bin/env.sh
-python -m ipykernel_launcher -f \$@
+# (3) also add the user's .local locations to allow package installation at runtime,
+# and the location of python modules installed in the base docker Collab image
+cat <<EOF >> $KERNEL_PATH/bin/env.sh
+export PATH=\$PATH:/opt/app-root/src/.local/bin
+export PYTHONPATH=\$PYTHONPATH:/opt/app-root/src/.local/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages
+export R_LIBS_USER=/opt/app-root/src/.local/lib/R/site-library
+mkdir -p \$R_LIBS_USER
+export R_LIBS=\$R_LIBS_USER:\$R_LIBS
 EOF
-chmod +x $KERNEL_PATH/bin/env.sh
 
-# create the new kernel's configuration file
+# (4) create startup script for python kernel
+cat $KERNEL_PATH/bin/env.sh > $KERNEL_PATH/bin/env_python.sh
+echo "python -m ipykernel_launcher -f \$@" >> $KERNEL_PATH/bin/env_python.sh
+chmod +x $KERNEL_PATH/bin/env_python.sh
+
+# ... and the new kernel's configuration file
 mkdir -p $KERNEL_PATH/$KERNEL_NAME
-cat <<EOF >$KERNEL_PATH/$KERNEL_NAME/kernel.json
+cat <<EOF > $KERNEL_PATH/$KERNEL_NAME/kernel.json
 {
- "argv": ["$KERNEL_PATH/bin/env.sh", "{connection_file}", "--profile=default"],
+ "argv": ["$KERNEL_PATH/bin/env_python.sh", "{connection_file}", "--profile=default"],
  "display_name": "$RELEASE_NAME",
- "name": "$KERNEL_NAME",
  "language": "python",
  "env": { "LAB_KERNEL_NAME": "$RELEASE_NAME", "LAB_KERNEL_RELEASE_DATE": "$(date +"%Y-%m-%d")" }
 }
 EOF
 
-# add EBRAINS logo to kernel
-cp $LAB_KERNEL_ROOT/../logo/logo-64x64.png $KERNEL_PATH/$KERNEL_NAME/
+# ... and add EBRAINS logo to kernel
+cp $LAB_KERNEL_ROOT/../etc/logo-64x64.png $KERNEL_PATH/$KERNEL_NAME/
+
+# (5) create startup script for R kernel
+cat $KERNEL_PATH/bin/env.sh > $KERNEL_PATH/bin/env_r.sh
+echo "R --slave -e \"IRkernel::main()\" --args \$@" >> $KERNEL_PATH/bin/env_r.sh
+chmod +x $KERNEL_PATH/bin/env_r.sh
+
+# ... and the new R kernel's configuration file
+mkdir -p $KERNEL_PATH/${KERNEL_NAME}-R
+cat <<EOF > $KERNEL_PATH/${KERNEL_NAME}-R/kernel.json
+{
+ "argv": ["$KERNEL_PATH/bin/env_r.sh", "{connection_file}"],
+ "display_name": "R-$RELEASE_NAME",
+ "language": "R",
+ "env": { "LAB_KERNEL_NAME": "R-$RELEASE_NAME", "LAB_KERNEL_RELEASE_DATE": "$(date +"%Y-%m-%d")" }
+}
+EOF
+
+# ... and add EBRAINS logo and kernel.js to kernel
+cp $LAB_KERNEL_ROOT/../etc/logo-64x64.png $KERNEL_PATH/${KERNEL_NAME}-R/
+cp $LAB_KERNEL_ROOT/../etc/kernel.js $KERNEL_PATH/${KERNEL_NAME}-R/

install_spack_env.sh

@@ -16,17 +16,17 @@ EBRAINS_SPACK_ENV=$4
 export SPACK_USER_CACHE_PATH=$INSTALLATION_ROOT/spack/.spack
 export SPACK_USER_CONFIG_PATH=$INSTALLATION_ROOT/spack/.spack
 
-# initial setup: if spack dir doesn't already exist, clone it and install gcc and python
+# initial setup: clone spack if spack dir doesn't already exist and activate
 if [ ! -d $INSTALLATION_ROOT/spack ]
 then
-  # clone spack repo, import packages.yaml config file and activate
   git clone --depth 1 -c advice.detachedHead=false -c feature.manyFiles=true --branch $SPACK_VERSION https://github.com/spack/spack $INSTALLATION_ROOT/spack
 fi
 cp $EBRAINS_REPO/packages.yaml $INSTALLATION_ROOT/spack/etc/spack/packages.yaml
 source $INSTALLATION_ROOT/spack/share/spack/setup-env.sh
+
 # install platform compiler and python (extract versions from packages.yaml)
 EBRAINS_SPACK_COMPILER=$(grep 'compiler' $EBRAINS_REPO/packages.yaml | awk -F'[][]' '{ print $2 }')
-EBRAINS_SPACK_PYTHON=python@$(grep -A1 'python' $EBRAINS_REPO/packages.yaml | tail -n1 | awk -F'[][]' '{ print $2 }')
+EBRAINS_SPACK_PYTHON=python@$(grep -A1 'python:$' $EBRAINS_REPO/packages.yaml | tail -n1 | awk -F'[][]' '{ print $2 }')
 spack compiler find
 spack install $EBRAINS_SPACK_COMPILER
 spack load $EBRAINS_SPACK_COMPILER
@@ -52,10 +52,16 @@ spack env activate $EBRAINS_SPACK_ENV
 spack concretize --fresh --force
 # install the environment, use 2 jobs to reduce the amount of required RAM
 spack install --fresh --no-check-signature -y -j2
-
-# create modules files with spack
-spack module tcl refresh -y
 # rebuild spack's database
 spack reindex
-# create loads script that when sourced activates and loads the installed spack environment
-spack env loads -r
+
+# create load script that when sourced activates and loads the installed spack environment, using views
+# this needs deactivating the environment first:
+spack env deactivate
+unset SPACK_LD_LIBRARY_PATH
+spack env activate --sh $EBRAINS_SPACK_ENV > $SPACK_ROOT/var/spack/environments/$EBRAINS_SPACK_ENV/load_env.sh
+
+# create modules files with spack
+# spack module tcl refresh -y
+# create loads script that when sourced activates and loads the installed spack environment, using modules
+# spack env loads -r

packages.yaml

@@ -4,6 +4,11 @@ packages:
         compiler: [gcc@10.3.0]
     python:
         version: [3.8.11]
+    r:
+        version: [4.1.3]
+        variants: [+X]
     py-torch:
         version: [1.11.0]
         variants: [~cuda~rocm~valgrind~mkldnn~mpi~gloo+tensorpipe~onnx_ml]
+    py-jax:
+        variants: [~cuda]

packages/bazel/add-include-limits-0.25.patch

+diff -pur third_party/ijar/mapped_file_unix.cc third_party/ijar/mapped_file_unix.cc
+--- third_party/ijar/mapped_file_unix.cc	2021-10-05 15:24:37.243946507 +0200
++++ third_party/ijar/mapped_file_unix.cc	2021-10-05 15:24:28.707823401 +0200
+@@ -19,6 +19,7 @@
+ #include <sys/mman.h>
+ 
+ #include <algorithm>
++#include <limits>
+ 
+ #include "third_party/ijar/mapped_file.h"
+ 

packages/bazel/add-include-limits-zlib_client.patch

+diff -pur third_party/ijar/zlib_client.h third_party/ijar/zlib_client.h
+--- third_party/ijar/zlib_client.h	2021-10-07 09:32:57.087301302 +0200
++++ third_party/ijar/zlib_client.h	1980-01-01 00:00:00.000000000 +0100
+@@ -16,6 +16,7 @@
+ #define THIRD_PARTY_IJAR_ZLIB_CLIENT_H_
+ 
+ #include <limits.h>
++#include <limits>
+ 
+ #include "third_party/ijar/common.h"
+ 

packages/bazel/bazelconfiguration-0.3.patch

+--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
++++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
+@@ -150,6 +150,13 @@
+       builder.put("PATH", null);
+       builder.put("LD_LIBRARY_PATH", null);
+     }
++
++    Map<String, String> spackEnv = System.getenv();
++    for (String envName : spackEnv.keySet()) {
++      if (envName.startsWith("SPACK_")) {
++        builder.put(envName, spackEnv.get(envName));
++      }
++    }
+   }
+ 
+   private static PathFragment determineShellExecutable(OS os, PathFragment fromOption) {

packages/bazel/bazelruleclassprovider-0.14.patch

+--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
++++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+@@ -168,6 +168,13 @@ public class BazelRuleClassProvider {
+       env.put("PATH", null);
+     }
+ 
++    Map<String, String> spackEnv = System.getenv();
++    for (String envName : spackEnv.keySet()) {
++      if (envName.startsWith("SPACK_")) {
++        env.put(envName, spackEnv.get(envName));
++      }
++    }
++
+     // Shell environment variables specified via options take precedence over the
+     // ones inherited from the fragments. In the long run, these fragments will
+     // be replaced by appropriate default rc files anyway.

packages/bazel/bazelruleclassprovider-0.25.patch

+--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
++++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+@@ -181,6 +181,13 @@ public class BazelRuleClassProvider {
+           env.put("PATH", null);
+         }
+ 
++        Map<String, String> spackEnv = System.getenv();
++        for (String envName : spackEnv.keySet()) {
++          if (envName.startsWith("SPACK_")) {
++            env.put(envName, spackEnv.get(envName));
++          }
++        }
++
+         // Shell environment variables specified via options take precedence over the
+         // ones inherited from the fragments. In the long run, these fragments will
+         // be replaced by appropriate default rc files anyway.

packages/bazel/bazelruleclassprovider_fj-0.25.patch

+--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
++++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+@@ -185,7 +185,7 @@ public class BazelRuleClassProvider {
+ 
+         Map<String, String> spackEnv = System.getenv();
+         for (String envName : spackEnv.keySet()) {
+-          if (envName.startsWith("SPACK_")) {
++          if ((envName.startsWith("SPACK_")) || (envName.equals("fcc_ENV")) || (envName.equals("FCC_ENV"))) {
+             env.put(envName, spackEnv.get(envName));
+           }
+         }

packages/bazel/blaze_util_posix-0.29.1.patch

+diff -Naur a/src/main/cpp/blaze_util_posix.cc b/src/main/cpp/blaze_util_posix.cc
+--- a/src/main/cpp/blaze_util_posix.cc	1980-01-01 00:00:00.000000000 -0800
++++ b/src/main/cpp/blaze_util_posix.cc	2022-06-30 23:34:08.000000000 -0700
+@@ -600,7 +600,7 @@
+   // Prefer OFD locks if available.  POSIX locks can be lost "accidentally"
+   // due to any close() on the lock file, and are not reliably preserved
+   // across execve() on Linux, which we need for --batch mode.
+-  if (fcntl(fd, F_OFD_SETLK, lock) == 0) return 0;
++  if (fcntl(fd, F_SETLK, lock) == 0) return 0;
+   if (errno != EINVAL) {
+     if (errno != EACCES && errno != EAGAIN) {
+       BAZEL_DIE(blaze_exit_code::LOCAL_ENVIRONMENTAL_ERROR)

packages/bazel/build-0.29.1.patch

+From 9c9d27561780bc56d9f0867e325c7421a94ee1cb Mon Sep 17 00:00:00 2001
+From: Harsh Bhatia <bhatia4@llnl.gov>
+Date: Tue, 15 Dec 2020 15:56:10 -0800
+Subject: [PATCH] https://github.com/bazelbuild/bazel/commit/ab62a6e097590dac5ec946ad7a796ea0e8593ae0 
+
+---
+ src/conditions/BUILD | 6 ++++++
+ third_party/BUILD    | 8 ++++++--
+ 2 files changed, 12 insertions(+), 2 deletions(-)
+
+diff --git a/src/conditions/BUILD b/src/conditions/BUILD
+index 2b28e28057..faa41a439d 100644
+--- a/src/conditions/BUILD
++++ b/src/conditions/BUILD
+@@ -10,6 +10,12 @@ filegroup(
+     visibility = ["//src:__pkg__"],
+ )
+ 
++config_setting(
++    name = "linux_ppc",
++    values = {"cpu": "ppc"},
++    visibility = ["//visibility:public"],
++)
++
+ config_setting(
+     name = "linux_x86_64",
+     values = {"cpu": "k8"},
+diff --git a/third_party/BUILD b/third_party/BUILD
+index 159006d741..4fcae54c00 100644
+--- a/third_party/BUILD
++++ b/third_party/BUILD
+@@ -523,12 +523,13 @@ UNNECESSARY_DYNAMIC_LIBRARIES = select({
+     "//src/conditions:darwin": "*.so *.dll",
+     "//src/conditions:darwin_x86_64": "*.so *.dll",
+     "//src/conditions:linux_x86_64": "*.jnilib *.dll",
++    "//src/conditions:linux_ppc": "*.so *.jnilib *.dll",
+     # The .so file is an x86 one, so we can just remove it if the CPU is not x86
+     "//src/conditions:arm": "*.so *.jnilib *.dll",
+     "//src/conditions:linux_aarch64": "*.so *.jnilib *.dll",
+     # Play it safe -- better have a big binary than a slow binary
+     # zip -d does require an argument. Supply something bogus.
+-    "//conditions:default": "*.bogusextension",
++    "//conditions:default": "",
+ })
+ 
+ # Remove native libraries that are for a platform different from the one we are
+@@ -537,7 +538,10 @@ genrule(
+     name = "filter_netty_dynamic_libs",
+     srcs = ["netty_tcnative/netty-tcnative-boringssl-static-2.0.24.Final.jar"],
+     outs = ["netty_tcnative/netty-tcnative-filtered.jar"],
+-    cmd = "cp $< $@ && zip -qd $@ " + UNNECESSARY_DYNAMIC_LIBRARIES,
++    cmd = "cp $< $@ && " +
++      # End successfully if there is nothing to be deleted from the archive
++      "if [ -n '" + UNNECESSARY_DYNAMIC_LIBRARIES + "' ]; then " +
++      "zip -qd $@ " + UNNECESSARY_DYNAMIC_LIBRARIES + "; fi",
+ )
+ 
+ java_import(
+-- 
+2.21.0 (Apple Git-122.2)
+

packages/bazel/cc_configure-0.3.0.patch

+--- a/tools/cpp/cc_configure.bzl
++++ b/tools/cpp/cc_configure.bzl
+@@ -173,8 +173,19 @@
+   else:
+     inc_dirs = result.stderr[index1 + 1:index2].strip()
+ 
+-  return [repository_ctx.path(_cxx_inc_convert(p))
+-          for p in inc_dirs.split("\n")]
++  default_inc_directories = [
++    repository_ctx.path(_cxx_inc_convert(p))
++    for p in inc_dirs.split("\n")
++  ]
++
++  env = repository_ctx.os.environ
++  if "SPACK_INCLUDE_DIRS" in env:
++    for path in env["SPACK_INCLUDE_DIRS"].split(":"):
++      default_inc_directories.append(
++        repository_ctx.path(_cxx_inc_convert(path))
++      )
++
++  return default_inc_directories
+ 
+ def _add_option_if_supported(repository_ctx, cc, option):
+   """Checks that `option` is supported by the C compiler."""

packages/bazel/cc_configure-0.5.0.patch

+--- a/tools/cpp/cc_configure.bzl
++++ b/tools/cpp/cc_configure.bzl
+@@ -200,8 +200,19 @@
+   else:
+     inc_dirs = result.stderr[index1 + 1:index2].strip()
+ 
+-  return [_escape_string(repository_ctx.path(_cxx_inc_convert(p)))
+-          for p in inc_dirs.split("\n")]
++  default_inc_directories = [
++    _escape_string(repository_ctx.path(_cxx_inc_convert(p)))
++    for p in inc_dirs.split("\n")
++  ]
++
++  env = repository_ctx.os.environ
++  if "SPACK_INCLUDE_DIRS" in env:
++    for path in env["SPACK_INCLUDE_DIRS"].split(":"):
++      default_inc_directories.append(
++        repository_ctx.path(_cxx_inc_convert(path))
++      )
++
++  return default_inc_directories
+ 
+ 
+ def _add_option_if_supported(repository_ctx, cc, option):

packages/bazel/cc_env.patch

+diff --git a/compile.sh b/compile.sh
+index ed506d2..a5474ac 100755
+--- a/compile.sh
++++ b/compile.sh
+@@ -92,7 +92,7 @@ display "."
+ log "Building output/bazel"
+ # We set host and target platform directly since the defaults in @bazel_tools
+ # have not yet been generated.
+-bazel_build "src:bazel_nojdk${EXE_EXT}" \
++CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel_nojdk${EXE_EXT}" \
+   --host_platform=@bazel_tools//platforms:host_platform \
+   --platforms=@bazel_tools//platforms:target_platform \
+   || fail "Could not build Bazel"

packages/bazel/compile-0.13.patch

+--- a/compile.sh
++++ b/compile.sh
+@@ -92,7 +92,7 @@
+ log "Building output/bazel"
+ # We set host and target platform directly since the defaults in @bazel_tools
+ # have not yet been generated.
+-bazel_build "src:bazel${EXE_EXT}" \
++CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel${EXE_EXT}" \
+   --host_platform=@bazel_tools//platforms:host_platform \
+   --platforms=@bazel_tools//platforms:target_platform \
+   || fail "Could not build Bazel"

packages/bazel/compile-0.16.patch

+--- a/compile.sh
++++ b/compile.sh
+@@ -92,7 +92,7 @@ display "."
+ log "Building output/bazel"
+ # We set host and target platform directly since the defaults in @bazel_tools
+ # have not yet been generated.
+-bazel_build "src:bazel_nojdk${EXE_EXT}" \
++CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel_nojdk${EXE_EXT}" \
+   --host_platform=@bazel_tools//platforms:host_platform \
+   --platforms=@bazel_tools//platforms:target_platform \
+   || fail "Could not build Bazel"

packages/bazel/compile-0.21.patch

+--- a/compile.sh
++++ b/compile.sh
+@@ -92,7 +92,7 @@ display "."
+ log "Building output/bazel"
+ # We set host and target platform directly since the defaults in @bazel_tools
+ # have not yet been generated.
+-bazel_build "src:bazel_nojdk${EXE_EXT}" \
++CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel_nojdk${EXE_EXT}" \
+   --action_env=PATH \
+   --host_platform=@bazel_tools//platforms:host_platform \
+   --platforms=@bazel_tools//platforms:target_platform \

packages/bazel/compile-0.29.patch

+--- a/compile.sh
++++ b/compile.sh
+@@ -63,7 +63,7 @@
+ log "Building output/bazel"
+ # We set host and target platform directly because we are building for the local
+ # host.
+-bazel_build "src:bazel_nojdk${EXE_EXT}" \
++CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel_nojdk${EXE_EXT}" \
+   --action_env=PATH \
+   --host_platform=@local_config_platform//:host \
+   --platforms=@local_config_platform//:host \

packages/bazel/compile-0.3.patch

+--- a/compile.sh
++++ b/compile.sh
+@@ -99,7 +99,7 @@
+   new_step 'Building Bazel with Bazel'
+   display "."
+   log "Building output/bazel"
+-  bazel_build "src:bazel${EXE_EXT}"
++  CC=$SPACK_CC CXX=$SPACK_CXX bazel_build "src:bazel${EXE_EXT}"
+   cp -f "bazel-bin/src/bazel${EXE_EXT}" "output/bazel${EXE_EXT}"
+   chmod 0755 "output/bazel${EXE_EXT}"
+   BAZEL="$(pwd)/output/bazel${EXE_EXT}"