stages:
- prepare
- frontend
- buildnode
variables:
GITLAB_BUILD_ENV_DOCKER_IMAGE: docker-registry.ebrains.eu/tc/ebrains-spack-build-env/gitlab_runners_nfs:gitlab_runners_nfs_23.06
YASHCHIKI_HOME: ${CI_PROJECT_DIR}/.yashchiki
http_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
https_proxy: "http://proxy.kip.uni-heidelberg.de:8080"
HTTP_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
HTTPS_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
ALL_PROXY: "http://proxy.kip.uni-heidelberg.de:8080"
SYSTEMNAME: "image_laptop"
SPACK_ENVIRONMENT_REPO: "${CI_PROJECT_DIR}/esd_spack/var/spack/repos/ebrains-spack-builds"
SPACK_ENVIRONMENT_PATH: "${CI_PROJECT_DIR}/esd_spack/var/spack/environments/default"
TMPDIR: "/tmp"
# the image build tool needs Python `yaml` and `apptainer` — we build it via spack
buildenv:
stage: prepare
tags:
- esd_image
image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
variables:
SPACK_DEV_ENV: ebrains-dev
script:
# FIXME: that's probably not what we want → we should always clone and install from buildcache!
- env
- date
- ls -lisa
- test -d esd_spack_buildenv || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack_buildenv
- pushd esd_spack_buildenv; git fetch origin HEAD && git reset --hard FETCH_HEAD; popd
- test -d esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack_buildenv/var/spack/repos/ebrains-spack-builds
- pushd esd_spack_buildenv/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
- esd_spack_buildenv/bin/spack repo add --scope=site esd_spack_buildenv/var/spack/repos/ebrains-spack-builds || true
- . esd_spack_buildenv/share/spack/setup-env.sh
- spack compiler find --scope=site /usr/bin
- spack external find --scope=site python
- spack bootstrap root esd_spack_buildenv/opt/spack/bootstrap
- spack bootstrap now
- date
- (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) py-pyyaml 2>&1 | sed -e "s:^:[py-pyyaml] :g") &
- (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) rsync 2>&1 | sed -e "s:^:[rsync] :g") &
- (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot 2>&1 | sed -e "s:^:[proot-0] :g") &
- (nohup spack install -j $(( ($(nproc) * 2 - 4) / 4 + 1)) proot 2>&1 | sed -e "s:^:[proot-1] :g") &
- wait
- date
- (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot 2>&1 | sed -e "s:^:[fakeroot-0] :g") &
- (nohup spack install -v -j $(( ($(nproc) * 2 - 2) / 2 + 1)) fakeroot 2>&1 | sed -e "s:^:[fakeroot-1] :g") &
- wait
- date
# some more parallelism for apptainer (and oversubscribe!)
- (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-0] :g") &
- (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-1] :g") &
- (nohup spack install -j $(( ($(nproc) * 2 - 3) / 3 + 1)) apptainer~suid 2>&1 | sed -e "s:^:[apptainer-2] :g") &
- wait
- date
- (nohup spack install -j $(( $(nproc) * 2 )) "skopeo@1.6:" 2>&1 | sed -e "s:^:[skopeo] :g") &
- wait
- date
- (nohup spack install -j $(( $(nproc) * 2 )) "oras" 2>&1 | sed -e "s:^:[oras] :g") &
- wait
- date
- (nohup spack install -j $(( $(nproc) * 2 )) "fakechroot" 2>&1 | sed -e "s:^:[fakechroot] :g") &
- wait
- date
- spack load "oras@1.1:"
cache:
key: buildenv-$CI_COMMIT_REF_SLUG
policy: pull-push
when: always
paths:
- esd_spack_buildenv/
timeout: 2 days
# fetch all sources for later build stages on some internet-connected CI runner
fetch:
stage: frontend
dependencies:
- buildenv
tags:
- esd_image
image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
variables:
SPACK_DEV_ENV: ebrains-dev
script:
- date
- ls -lisa
- . esd_spack_buildenv/share/spack/setup-env.sh
- spack load py-pyyaml rsync "oras@1.1:" proot
- rm -rf esd_spack
- test -d esd_spack || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack
- pushd esd_spack; git fetch origin eric_testing && git reset --hard FETCH_HEAD; popd
- test -d esd_spack/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack/var/spack/repos/ebrains-spack-builds
- pushd esd_spack/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
- echo "repos:" > esd_spack/etc/spack/repos.yaml
- echo " - \$spack/var/spack/repos/ebrains-spack-builds" >> esd_spack/etc/spack/repos.yaml
- cat esd_spack/etc/spack/repos.yaml
- esd_spack/bin/spack repo list
- esd_spack/bin/spack repo list --scope=site
- proot -b share/yashchiki/styles/esd/fetch_os-release:/etc/os-release python3 bin/yashchiki --debug --stages fetch -- esd esd_spack esd_output
- date
cache:
- key: buildenv-$CI_COMMIT_REF_SLUG
policy: pull
paths:
- esd_spack_buildenv/
- key: fetch-$CI_COMMIT_REF_SLUG
policy: pull-push
when: always
paths:
- esd_spack/
- .yashchiki/download_cache
artifacts:
when: always
paths:
- errors_concretization.log
- ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
- ${YASHCHIKI_HOME}/log
- /tmp/tmp.*/spec_*.yaml*
timeout: 2 days
# create the base image for the spack build
build-base-image:
stage: frontend
dependencies:
- buildenv
tags:
- esd_image
image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
variables:
SPACK_DEV_ENV: ebrains-dev
script:
- date
- ls -lisa
- . esd_spack_buildenv/share/spack/setup-env.sh
- spack load py-pyyaml rsync apptainer~suid proot fakeroot "skopeo@1.6:" "oras@1.1:"
# * inspect underlying base image → determine identifier for specific version
- HASH_BASE_DISTRO=$(skopeo inspect docker://debian:bookworm | sha256sum)
# * hash build base image-related things (FIXME: this should be done differently ;))
- HASH_BUILD_BASE_IMAGE=$(sha256sum lib/yashchiki/build_base_sandbox.sh)
- HASH_BUILD_BASE_IMAGE_CONTENT=$(sha256sum share/yashchiki/styles/esd/install_prerequisites.sh)
# * combine all hashes into one hash → use as lookup into gitlab package registry
- ESD_BASE_HASH=$(echo ${HASH_BASE_DISTRO}${HASH_BUILD_BASE_IMAGE}${HASH_BUILD_BASE_IMAGE_CONTENT} | sha256sum | cut -d\ -f 1)
- echo ${ESD_BASE_HASH} | tee esd_base_hash
# * try to download base build image, else build image and upload image into package registry
- set -x
- skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} && ret=$? || ret=$?
- date
- |
if [ "$ret" -ne 0 ]; then
python3 bin/yashchiki --stages build-base-image -- esd esd_spack esd_output;
date
skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} || true
date
# replace symbolic tag
skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base || true
else
# needs to be an artifact => provide to build node via gitlab mechanisms
apptainer build esd_output_base docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_base_${ESD_BASE_HASH}
fi
- apptainer -d exec --fakeroot --no-pid esd_output_base bash -c "hostname" || true
- date
cache:
- key: buildenv-$CI_COMMIT_REF_SLUG
policy: pull
paths:
- esd_spack_buildenv/
artifacts:
when: always
paths:
- ${YASHCHIKI_HOME}/log
- esd_base_hash
# requires for downstream job…
- esd_output_base
timeout: 2 days
# use the base image and build software in there using spack
buildnode-stuff:
stage: buildnode
dependencies:
- fetch
- build-base-image
tags:
- esd_image
image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
variables:
SPACK_DEV_ENV: ebrains-dev
script:
- date
- du -sh esd_spack_buildenv/
- date
- ls -lisa
- . esd_spack_buildenv/share/spack/setup-env.sh
- spack load py-pyyaml rsync apptainer~suid proot fakeroot fakechroot "skopeo@1.6:" "oras@1.1:"
- HASH_ESD_BASE_IMAGE=$(cat esd_base_hash)
- git config --global --add safe.directory $PWD
- HASH_ESD_CONFIG=$(git ls-files -s share/yashchiki/styles/esd | git hash-object --stdin)
- HASH_ESD_SPACK=$(GIT_DIR=esd_spack/.git git rev-parse HEAD)
- HASH_ESD_REPO=$(GIT_DIR=esd_spack/var/spack/repos/ebrains-spack-builds/.git git rev-parse HEAD)
- echo ${HASH_ESD_BASE_IMAGE} ${HASH_ESD_CONFIG} ${HASH_ESD_SPACK} ${HASH_ESD_REPO}
- echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin
- ESD_HASH=$(echo ${HASH_ESD_BASE_IMAGE}${HASH_ESD_CONFIG}${HASH_ESD_SPACK}${HASH_ESD_REPO} | git hash-object --stdin)
- set -x
- skopeo inspect docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} && ret=$? || ret=$?
- date
- |
if [ "$ret" -ne 0 ]; then
python3 bin/yashchiki --debug --jobs $(nproc) --update-build-cache --stages build-base build-spack image -- esd esd_spack esd_output
date
find /tmp -name "spack-build-out.txt" -exec || true
find /tmp -name "spack-build-out.txt" -print0 | tar -cvzf spack-build-outs.tar.gz --null --files-from -
find "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" -maxdepth 2 -exec ls -l {} \; || true
du -sh "${YASHCHIKI_HOME}/sandboxes/esd/opt/spack/opt/spack" || true
# upload ESD image
skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" sif:esd_output docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} || true
date
# add a second tag (symbolic tag, potentially replacing an old one)
skopeo copy --dest-username="$HARBOR_USERNAME" --dest-password="$HARBOR_PASSWORD" docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output_${ESD_HASH} docker://${HARBOR_HOST}/${HARBOR_PROJECT}/esd:output || true
fi
- date
cache:
- key: buildenv-$CI_COMMIT_REF_SLUG
policy: pull
paths:
- esd_spack_buildenv/
- key: fetch-$CI_COMMIT_REF_SLUG
policy: pull
paths:
- esd_spack/
- .yashchiki/
- key: buildnode-stuff-$CI_COMMIT_REF_SLUG
policy: pull-push
when: always
paths:
- .yashchiki/build_caches
- .yashchiki/preserved_packages
- .yashchiki/spack_ccache
artifacts:
when: always
paths:
- errors_concretization.log
- ${YASHCHIKI_HOME}/sandboxes/esd/opt/spack_specs
- ${YASHCHIKI_HOME}/log
- spack-build-outs.tar.gz
timeout: 5 days
test-oci-buildcache:
when: manual
dependencies:
- buildenv
stage: frontend
tags:
- esd_image
image: $GITLAB_BUILD_ENV_DOCKER_IMAGE
variables:
SPACK_DEV_ENV: ebrains-dev
script:
- date
- ls -lisa
- . esd_spack_buildenv/share/spack/setup-env.sh
- spack load py-pyyaml rsync
- rm -rf esd_spack
- test -d esd_spack || git clone -b eric_testing https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/spack esd_spack
- pushd esd_spack; git fetch origin eric_testing && git reset --hard FETCH_HEAD; popd
- test -d esd_spack/var/spack/repos/ebrains-spack-builds || git clone -b image_build https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds esd_spack/var/spack/repos/ebrains-spack-builds
- pushd esd_spack/var/spack/repos/ebrains-spack-builds; git fetch origin image_build && git reset --hard FETCH_HEAD; popd
- echo "repos:" > esd_spack/etc/spack/repos.yaml
- echo " - \$spack/var/spack/repos/ebrains-spack-builds" >> esd_spack/etc/spack/repos.yaml
- esd_spack/bin/spack repo list
- esd_spack/bin/spack repo list --scope=site
- echo "mirrors:" > esd_spack/etc/spack/mirrors.yaml
- echo " ebrains_harbor:" >> esd_spack/etc/spack/mirrors.yaml
- echo " url:" "oci://docker-registry.ebrains.eu/brainscales/esd_buildcache_spackformat" >> esd_spack/etc/spack/mirrors.yaml
- . esd_spack/share/spack/setup-env.sh
- spack mirror list
- spack mirror list --scope=site
- spack buildcache list
- spack env create default
- rsync -a ${SPACK_ENVIRONMENT_REPO}/. esd_spack/var/spack/environments/default
- spack-python ${SPACK_ENVIRONMENT_REPO}/site-config/ymerge.py esd_spack/var/spack/environments/default/spack.yaml ${SPACK_ENVIRONMENT_REPO}/site-config/${SYSTEMNAME}/spack.yaml > /tmp/spack.yaml
- mv /tmp/spack.yaml esd_spack/var/spack/environments/default
- date
- spack -e default --test=root --fresh concretize
- date
- spack -e default spec
- date
- spack -e default install --use-buildcache=only
- date
cache:
- key: buildenv-$CI_COMMIT_REF_SLUG
policy: pull
paths:
- esd_spack_buildenv/