diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..7ad07e60d491d315ad614e51fbf06258baaa03c0
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,251 @@
+stages:
+ - prepare
+ - frontend
+ - buildnode
+
+
+variables:
+ GITLAB_BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:gitlab_runners_nfs_23.06
+ YASHCHIKI_HOME: ${CI_PROJECT_DIR}/.yashchiki
+ http_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
+ https_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
+ HTTP_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+ HTTPS_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+ ALL_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+ SYSTEMNAME: "image_laptop"
+ SPACK_ENVIRONMENT_REPO: "${CI_PROJECT_DIR}/esd_spack/var/spack/repos/ebrains-spack-builds"
+ SPACK_ENVIRONMENT_PATH: "${CI_PROJECT_DIR}/esd_spack/var/spack/environments/default"
+ TMPDIR: "/tmp"
+
+# the image build tool needs Python `yaml` and `apptainer` — we build it via spack
+buildenv:
+ stage: prepare
+ tags:
+ - esd_image
+ image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+ variables:
+ SPACK_DEV_ENV: ebrains-dev
+ script:
+ # FIXME: that's probably not what we want → we should always clone and install from buildcache!
+ - env
+ - date
+ - ls -lisa
+ - test -d esd_spack_buildenv || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack_buildenv
+ - pushd esd_spack_buildenv; git fetch origin HEAD && git reset --hard FETCH_HEAD; popd
+ - test -d esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack_buildenv/var/spack/repos/ebrains-spack-builds
+ - pushd esd_spack_buildenv/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
+ - esd_spack_buildenv/bin/spack repo add --scope=site esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || true
+ - . esd_spack_buildenv/share/spack/setup-env.sh
+ - spack compiler find --scope=site /usr/bin
+ - spack external find --scope=site python
+ - spack bootstrap root esd_spack_buildenv/opt/spack/bootstrap
+ - spack bootstrap now
+ - date
+ - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) py-pyyaml 2>&1 | sed -e "s:^:[py-pyyaml] :g") &
+ - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) rsync 2>&1 | sed -e "s:^:[rsync] :g") &
+ - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot 2>&1 | sed -e "s:^:[proot-0] :g") &
+ - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot 2>&1 | sed -e "s:^:[proot-1] :g") &
+ - wait
+ - date
+ - (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot 2>&1 | sed -e "s:^:[fakeroot-0] :g") &
+ - (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot 2>&1 | sed -e "s:^:[fakeroot-1] :g") &
+ - wait
+ - date
+ # some more parallelism for apptainer (and oversubscribe!)
+ - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-0] :g") &
+ - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-1] :g") &
+ - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-2] :g") &
+ - wait
+ - date
+ - (nohup spack install -j $(( $(nproc) * 2 )) "skopeo@1.6:" 2>&1 | sed -e "s:^:[skopeo] :g") &
+ - wait
+ - date
+ - (nohup spack install -j $(( $(nproc) * 2 )) "oras" 2>&1 | sed -e "s:^:[oras] :g") &
+ - wait
+ - date
+ - (nohup spack install -j $(( $(nproc) * 2 )) "fakechroot" 2>&1 | sed -e "s:^:[fakechroot] :g") &
+ - wait
+ - date
+ - spack load "oras@1.1:"
+ cache:
+ key: buildenv-$CI_COMMIT_REF_SLUG
+ policy: pull-push
+ when: always
+ paths:
+ - esd_spack_buildenv/
+ timeout: 2 days
+
+
+# fetch all sources for later build stages on some internet-connected CI runner
+fetch:
+ stage: frontend
+ dependencies:
+ - buildenv
+ tags:
+ - esd_image
+ image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+ variables:
+ SPACK_DEV_ENV: ebrains-dev
+ script:
+ - date
+ - ls -lisa
+ - . esd_spack_buildenv/share/spack/setup-env.sh
+ - spack load py-pyyaml rsync "oras@1.1:" proot
+ - rm -rf esd_spack
+ - test -d esd_spack || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack
+ - pushd esd_spack; git fetch origin eric_testing && git reset --hard FETCH_HEAD; popd
+ - test -d esd_spack/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack/var/spack/repos/ebrains-spack-builds
+ - pushd esd_spack/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
+ - echo "repos:" > esd_spack/etc/spack/repos.yaml
+ - echo " - \$spack/var/spack/repos/ebrains-spack-builds" >> esd_spack/etc/spack/repos.yaml
+ - cat esd_spack/etc/spack/repos.yaml
+ - esd_spack/bin/spack repo list
+ - esd_spack/bin/spack repo list --scope=site
+ - proot -b share/yashchiki/styles/esd/fetch_os-release:/etc/os-release python3 bin/yashchiki --debug --stages fetch -- esd esd_spack esd_output
+ - date
+ cache:
+ - key: buildenv-$CI_COMMIT_REF_SLUG
+ policy: pull
+ paths:
+ - esd_spack_buildenv/
+ - key: fetch-$CI_COMMIT_REF_SLUG
+ policy: pull-push
+ when: always
+ paths:
+ - esd_spack/
+ - .yashchiki/download_cache
+ artifacts:
+ when: always
+ paths:
+ - errors_concretization.log
+ - ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
+ - ${YASHCHIKI_HOME}/log
+ - /tmp/tmp.*/spec_*.yaml*
+ timeout: 2 days
+
+
+# create the base image for the spack build
+build-base-image:
+ stage: frontend
+ dependencies:
+ - buildenv
+ tags:
+ - esd_image
+ image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+ variables:
+ SPACK_DEV_ENV: ebrains-dev
+ script:
+ - date
+ - ls -lisa
+ - . esd_spack_buildenv/share/spack/setup-env.sh
+ - spack load py-pyyaml rsync apptainer~suid proot fakeroot "skopeo@1.6:" "oras@1.1:"
+ # * inspect underlying base image → determine identifier for specific version
+ - HASH_BASE_DISTRO=$(skopeo inspect docker://debian:bookworm | sha256sum)
+ # * hash build base image-related things (FIXME: this should be done differently ;))
+ - HASH_BUILD_BASE_IMAGE=$(sha256sum lib/yashchiki/build_base_sandbox.sh)
+ - HASH_BUILD_BASE_IMAGE_CONTENT=$(sha256sum share/yashchiki/styles/esd/install_prerequisites.sh)
+ # * combine all hashes into one hash → use as lookup into gitlab package registry
+ - ESD_BASE_HASH=$(echo ${HASH_BASE_DISTRO}${HASH_BUILD_BASE_IMAGE}${HASH_BUILD_BASE_IMAGE_CONTENT} | sha256sum | cut -d\ -f 1)
+ - echo ${ESD_BASE_HASH} | tee esd_base_hash
+ # * try to download base build image, else build image and upload image into package registry
+ - set -x
+ - skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} && ret=$? || ret=$?
+ - date
+ - |
+ if [ "$ret" -ne 0 ]; then
+ python3 bin/yashchiki --stages build-base-image -- esd esd_spack esd_output;
+ date
+ skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} || true
+ date
+ # replace symbolic tag
+ skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base || true
+ else
+ # needs to be an artifact => provide to build node via gitlab mechanisms
+ apptainer build esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH}
+ fi
+ - apptainer -d exec --fakeroot --no-pid esd_output_base bash -c "hostname" || true
+ - date
+ cache:
+ - key: buildenv-$CI_COMMIT_REF_SLUG
+ policy: pull
+ paths:
+ - esd_spack_buildenv/
+ artifacts:
+ when: always
+ paths:
+ - ${YASHCHIKI_HOME}/log
+ - esd_base_hash
+ # requires for downstream job…
+ - esd_output_base
+ timeout: 2 days
+
+
+# use the base image and build software in there using spack
+buildnode-stuff:
+ stage: buildnode
+ dependencies:
+ - fetch
+ - build-base-image
+ tags:
+ - esd_image
+ image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+ variables:
+ SPACK_DEV_ENV: ebrains-dev
+ script:
+ - date
+ - du -sh esd_spack_buildenv/
+ - date
+ - ls -lisa
+ - . esd_spack_buildenv/share/spack/setup-env.sh
+ - spack load py-pyyaml rsync apptainer~suid proot fakeroot fakechroot "skopeo@1.6:" "oras@1.1:"
+ - HASH_ESD_BASE_IMAGE=$(cat esd_base_hash)
+ - git config --global --add safe.directory $PWD
+ - HASH_ESD_CONFIG=$(git ls-files -s share/yashchiki/styles/esd | git hash-object --stdin)
+ - HASH_ESD_SPACK=$(GIT_DIR=esd_spack/.git git rev-parse HEAD)
+ - HASH_ESD_REPO=$(GIT_DIR=esd_spack/var/spack/repos/ebrains-spack-builds/.git git rev-parse HEAD)
+ - echo ${HASH_ESD_BASE_IMAGE} ${HASH_ESD_CONFIG} ${HASH_ESD_SPACK} ${HASH_ESD_REPO}
+ - echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin
+ - ESD_HASH=$(echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin)
+ - set -x
+ - skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} && ret=$? || ret=$?
+ - date
+ - |
+ if [ "$ret" -ne 0 ]; then
+ python3 bin/yashchiki --debug --jobs $(nproc) --update-build-cache --stages build-base build-spack image -- esd esd_spack esd_output
+ date
+ find /tmp -name "spack-build-out.txt" -exec || true
+ find /tmp -name "spack-build-out.txt" -print0 | tar -cvzf spack-build-outs.tar.gz --null --files-from -
+ find "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" -maxdepth 2 -exec ls -l {} \; || true
+ du -sh "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" || true
+ # upload ESD image
+ skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} || true
+ date
+ # add a second tag (symbolic tag, potentially replacing an old one)
+ skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output || true
+ fi
+ - date
+ cache:
+ - key: buildenv-$CI_COMMIT_REF_SLUG
+ policy: pull
+ paths:
+ - esd_spack_buildenv/
+ - key: fetch-$CI_COMMIT_REF_SLUG
+ policy: pull
+ paths:
+ - esd_spack/
+ - .yashchiki/
+ - key: buildnode-stuff-$CI_COMMIT_REF_SLUG
+ policy: pull-push
+ when: always
+ paths:
+ - .yashchiki/build_caches
+ - .yashchiki/preserved_packages
+ - .yashchiki/spack_ccache
+ artifacts:
+ when: always
+ paths:
+ - errors_concretization.log
+ - ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
+ - ${YASHCHIKI_HOME}/log
+ - spack-build-outs.tar.gz
+ timeout: 5 days
diff --git a/lib/yashchiki/build_sandbox.sh b/lib/yashchiki/build_sandbox.sh
index 14723b1e5b237c1dab76a815d0382ea7dd299912..63e82755bce976a48b4b9708058d0d3f1e1314d8 100755
--- a/lib/yashchiki/build_sandbox.sh
+++ b/lib/yashchiki/build_sandbox.sh
@@ -102,6 +102,10 @@ export CONTAINER_STYLE="${CONTAINER_STYLE}"
"${SPACK_INSTALL_SCRIPTS}/complete_spack_install_routine_called_in_post.sh"
EOF
+# test
+echo apptainer -d exec --fakeroot ${TARGET_FOLDER} bash -c "hostname"
+apptainer -d exec --fakeroot ${TARGET_FOLDER} bash -c "hostname" || true
+
PROOT_NO_SECCOMP=1 proot -w / -S ${TARGET_FOLDER} \
--bind=${YASHCHIKI_CACHES_ROOT}/download_cache:/opt/spack/var/spack/cache \
--bind=${YASHCHIKI_CACHES_ROOT}/spack_ccache:/opt/ccache \