feat(gitlab): add basic CI script [DO-NOT-MERGE]

Change-Id: I8dc9aa427524c9662d7a23bbd04ef0589279ad14

.gitlab-ci.yml

+stages:
+  - prepare
+  - frontend
+  - buildnode
+
+
+variables:
+  GITLAB_BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:gitlab_runners_nfs_23.06
+  YASHCHIKI_HOME: ${CI_PROJECT_DIR}/.yashchiki
+  http_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
+  https_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
+  HTTP_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+  HTTPS_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+  ALL_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
+  SYSTEMNAME: "image_laptop"
+  SPACK_ENVIRONMENT_REPO: "${CI_PROJECT_DIR}/esd_spack/var/spack/repos/ebrains-spack-builds"
+  SPACK_ENVIRONMENT_PATH: "${CI_PROJECT_DIR}/esd_spack/var/spack/environments/default"
+  TMPDIR: "/tmp"
+
+# the image build tool needs Python `yaml` and `apptainer` — we build it via spack
+buildenv:
+  stage: prepare
+  tags:
+    - esd_image
+  image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+  variables:
+    SPACK_DEV_ENV: ebrains-dev
+  script:
+    # FIXME: that's probably not what we want → we should always clone and install from buildcache!
+    - env
+    - date
+    - ls -lisa
+    - test -d esd_spack_buildenv || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack_buildenv
+    - pushd esd_spack_buildenv; git fetch origin HEAD && git reset --hard FETCH_HEAD; popd
+    - test -d esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack_buildenv/var/spack/repos/ebrains-spack-builds
+    - pushd esd_spack_buildenv/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
+    - esd_spack_buildenv/bin/spack repo add --scope=site esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || true
+    - . esd_spack_buildenv/share/spack/setup-env.sh
+    - spack compiler find --scope=site /usr/bin
+    - spack external find --scope=site python
+    - spack bootstrap root esd_spack_buildenv/opt/spack/bootstrap
+    - spack bootstrap now
+    - date
+    - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) py-pyyaml 2>&1 | sed -e "s:^:[py-pyyaml] :g") &
+    - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) rsync     2>&1 | sed -e "s:^:[rsync] :g") &
+    - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot     2>&1 | sed -e "s:^:[proot-0] :g") &
+    - (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot     2>&1 | sed -e "s:^:[proot-1] :g") &
+    - wait
+    - date
+    - (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot  2>&1 | sed -e "s:^:[fakeroot-0] :g") &
+    - (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot  2>&1 | sed -e "s:^:[fakeroot-1] :g") &
+    - wait
+    - date
+    # some more parallelism for apptainer (and oversubscribe!)
+    - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-0] :g") &
+    - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-1] :g") &
+    - (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-2] :g") &
+    - wait
+    - date
+    - (nohup spack install -j $(( $(nproc) * 2 )) "skopeo@1.6:" 2>&1 | sed -e "s:^:[skopeo] :g") &
+    - wait
+    - date
+    - (nohup spack install -j $(( $(nproc) * 2 )) "oras" 2>&1 | sed -e "s:^:[oras] :g") &
+    - wait
+    - date
+    - (nohup spack install -j $(( $(nproc) * 2 )) "fakechroot" 2>&1 | sed -e "s:^:[fakechroot] :g") &
+    - wait
+    - date
+    - spack load "oras@1.1:"
+  cache:
+    key: buildenv-$CI_COMMIT_REF_SLUG
+    policy: pull-push
+    when: always
+    paths:
+      - esd_spack_buildenv/
+  timeout: 2 days
+
+
+# fetch all sources for later build stages on some internet-connected CI runner
+fetch:
+  stage: frontend
+  dependencies:
+    - buildenv
+  tags:
+    - esd_image
+  image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+  variables:
+    SPACK_DEV_ENV: ebrains-dev
+  script:
+    - date
+    - ls -lisa
+    - . esd_spack_buildenv/share/spack/setup-env.sh
+    - spack load py-pyyaml rsync "oras@1.1:" proot
+    - rm -rf esd_spack
+    - test -d esd_spack || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack
+    - pushd esd_spack; git fetch origin eric_testing && git reset --hard FETCH_HEAD; popd
+    - test -d esd_spack/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack/var/spack/repos/ebrains-spack-builds
+    - pushd esd_spack/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
+    - echo "repos:" > esd_spack/etc/spack/repos.yaml
+    - echo "  - \$spack/var/spack/repos/ebrains-spack-builds" >> esd_spack/etc/spack/repos.yaml
+    - cat esd_spack/etc/spack/repos.yaml
+    - esd_spack/bin/spack repo list
+    - esd_spack/bin/spack repo list --scope=site
+    - proot -b share/yashchiki/styles/esd/fetch_os-release:/etc/os-release python3 bin/yashchiki --debug --stages fetch -- esd esd_spack esd_output
+    - date
+  cache:
+    - key: buildenv-$CI_COMMIT_REF_SLUG
+      policy: pull
+      paths:
+        - esd_spack_buildenv/
+    - key: fetch-$CI_COMMIT_REF_SLUG
+      policy: pull-push
+      when: always
+      paths:
+        - esd_spack/
+        - .yashchiki/download_cache
+  artifacts:
+    when: always
+    paths:
+      - errors_concretization.log
+      - ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
+      - ${YASHCHIKI_HOME}/log
+      - /tmp/tmp.*/spec_*.yaml*
+  timeout: 2 days
+
+
+# create the base image for the spack build
+build-base-image:
+  stage: frontend
+  dependencies:
+    - buildenv
+  tags:
+    - esd_image
+  image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+  variables:
+    SPACK_DEV_ENV: ebrains-dev
+  script:
+    - date
+    - ls -lisa
+    - . esd_spack_buildenv/share/spack/setup-env.sh
+    - spack load py-pyyaml rsync apptainer~suid proot fakeroot "skopeo@1.6:" "oras@1.1:"
+    # * inspect underlying base image → determine identifier for specific version
+    - HASH_BASE_DISTRO=$(skopeo inspect docker://debian:bookworm | sha256sum)
+    # * hash build base image-related things (FIXME: this should be done differently ;))
+    - HASH_BUILD_BASE_IMAGE=$(sha256sum lib/yashchiki/build_base_sandbox.sh)
+    - HASH_BUILD_BASE_IMAGE_CONTENT=$(sha256sum share/yashchiki/styles/esd/install_prerequisites.sh)
+    # * combine all hashes into one hash → use as lookup into gitlab package registry
+    - ESD_BASE_HASH=$(echo ${HASH_BASE_DISTRO}${HASH_BUILD_BASE_IMAGE}${HASH_BUILD_BASE_IMAGE_CONTENT} | sha256sum | cut -d\  -f 1)
+    - echo ${ESD_BASE_HASH} | tee esd_base_hash
+    # * try to download base build image, else build image and upload image into package registry
+    - set -x
+    - skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} && ret=$? || ret=$?
+    - date
+    - |
+      if [ "$ret" -ne 0 ]; then
+          python3 bin/yashchiki --stages build-base-image -- esd esd_spack esd_output;
+          date
+          skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} || true
+          date
+          # replace symbolic tag
+          skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base || true
+      else
+          # needs to be an artifact => provide to build node via gitlab mechanisms
+          apptainer build esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH}
+      fi
+    - apptainer -d exec --fakeroot --no-pid esd_output_base bash -c "hostname" || true
+    - date
+  cache:
+    - key: buildenv-$CI_COMMIT_REF_SLUG
+      policy: pull
+      paths:
+        - esd_spack_buildenv/
+  artifacts:
+    when: always
+    paths:
+      - ${YASHCHIKI_HOME}/log
+      - esd_base_hash
+      # requires for downstream job…
+      - esd_output_base
+  timeout: 2 days
+
+
+# use the base image and build software in there using spack
+buildnode-stuff:
+  stage: buildnode
+  dependencies:
+    - fetch
+    - build-base-image
+  tags:
+    - esd_image
+  image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
+  variables:
+    SPACK_DEV_ENV: ebrains-dev
+  script:
+    - date
+    - du -sh esd_spack_buildenv/
+    - date
+    - ls -lisa
+    - . esd_spack_buildenv/share/spack/setup-env.sh
+    - spack load py-pyyaml rsync apptainer~suid proot fakeroot fakechroot "skopeo@1.6:" "oras@1.1:"
+    - HASH_ESD_BASE_IMAGE=$(cat esd_base_hash)
+    - git config --global --add safe.directory $PWD
+    - HASH_ESD_CONFIG=$(git ls-files -s share/yashchiki/styles/esd | git hash-object --stdin)
+    - HASH_ESD_SPACK=$(GIT_DIR=esd_spack/.git git rev-parse HEAD)
+    - HASH_ESD_REPO=$(GIT_DIR=esd_spack/var/spack/repos/ebrains-spack-builds/.git git rev-parse HEAD)
+    - echo ${HASH_ESD_BASE_IMAGE} ${HASH_ESD_CONFIG} ${HASH_ESD_SPACK} ${HASH_ESD_REPO}
+    - echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin
+    - ESD_HASH=$(echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin)
+    - set -x
+    - skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} && ret=$? || ret=$?
+    - date
+    - |
+      if [ "$ret" -ne 0 ]; then
+          python3 bin/yashchiki --debug --jobs $(nproc) --update-build-cache --stages build-base build-spack image -- esd esd_spack esd_output
+          date
+          find /tmp -name "spack-build-out.txt" -exec || true
+          find /tmp -name "spack-build-out.txt" -print0 | tar -cvzf spack-build-outs.tar.gz --null --files-from -
+          find "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" -maxdepth 2 -exec ls -l {} \; || true
+          du -sh "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" || true
+          # upload ESD image
+          skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} || true
+          date
+          # add a second tag (symbolic tag, potentially replacing an old one)
+          skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output || true
+      fi
+    - date
+  cache:
+    - key: buildenv-$CI_COMMIT_REF_SLUG
+      policy: pull
+      paths:
+        - esd_spack_buildenv/
+    - key: fetch-$CI_COMMIT_REF_SLUG
+      policy: pull
+      paths:
+        - esd_spack/
+        - .yashchiki/
+    - key: buildnode-stuff-$CI_COMMIT_REF_SLUG
+      policy: pull-push
+      when: always
+      paths:
+        - .yashchiki/build_caches
+        - .yashchiki/preserved_packages
+        - .yashchiki/spack_ccache
+  artifacts:
+    when: always
+    paths:
+      - errors_concretization.log
+      - ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
+      - ${YASHCHIKI_HOME}/log
+      - spack-build-outs.tar.gz
+  timeout: 5 days

lib/yashchiki/build_sandbox.sh

@@ -102,6 +102,10 @@ export CONTAINER_STYLE="${CONTAINER_STYLE}"
 "${SPACK_INSTALL_SCRIPTS}/complete_spack_install_routine_called_in_post.sh"
 EOF
 
+# test
+echo apptainer -d exec --fakeroot ${TARGET_FOLDER} bash -c "hostname"
+apptainer -d exec --fakeroot ${TARGET_FOLDER} bash -c "hostname" || true
+
 PROOT_NO_SECCOMP=1 proot -w / -S ${TARGET_FOLDER} \
     --bind=${YASHCHIKI_CACHES_ROOT}/download_cache:/opt/spack/var/spack/cache \
     --bind=${YASHCHIKI_CACHES_ROOT}/spack_ccache:/opt/ccache \