From c06441e5f382d7e8f811d43c534b20047a11c98f Mon Sep 17 00:00:00 2001
From: Philipp Spilger <philipp.spilger@kip.uni-heidelberg.de>
Date: Wed, 8 Sep 2021 17:15:53 +0200
Subject: [PATCH] Add builder script for standalone usage and encapsulation
Change-Id: I6132b0443b084aec1dbefe529efd81cdba3ed377
---
.ci/Jenkinsfile | 70 ++----
.ci/Jenkinsfile_asic | 67 ++---
bin/yashchiki | 248 +++++++++++++++++++
bin/yashchiki_dump_meta_info.sh | 20 +-
lib/yashchiki/build_image.sh | 5 +
lib/yashchiki/commons.sh | 2 +-
lib/yashchiki/create_caches.sh | 28 +++
lib/yashchiki/create_spack_user.sh | 9 +
lib/yashchiki/gerrit.sh | 105 ++++++++
lib/yashchiki/restore_host_user_ownership.sh | 12 +
lib/yashchiki/update_build_cache.sh | 6 -
11 files changed, 467 insertions(+), 105 deletions(-)
create mode 100644 bin/yashchiki
create mode 100755 lib/yashchiki/create_caches.sh
create mode 100755 lib/yashchiki/create_spack_user.sh
create mode 100755 lib/yashchiki/gerrit.sh
create mode 100755 lib/yashchiki/restore_host_user_ownership.sh
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 3ed48998..7f5fca53 100755
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -28,30 +28,21 @@ pipeline {
}
environment {
- CONTAINER_STYLE = "visionary"
YASHCHIKI_HOST_ENV_PATH = "${WORKSPACE}/host.env"
}
stages {
stage('Container Build') {
+ // TODO: remove once unused
environment {
- DOCKER_BASE_IMAGE = "debian:bullseye"
- DEPENDENCY_PYTHON = "python@3.8.2"
+ CONTAINER_STYLE = "visionary"
YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
- YASHCHIKI_RECIPE_PATH = "${WORKSPACE}/visionary_recipe.def"
YASHCHIKI_CACHES_ROOT = "${HOME}"
YASHCHIKI_SPACK_PATH = "${env.WORKSPACE}/spack"
- YASHCHIKI_IMAGE_NAME = "singularity_visionary_temp.img"
YASHCHIKI_SANDBOXES = "sandboxes"
YASHCHIKI_PROXY_HTTP = "http://proxy.kip.uni-heidelberg.de:8080"
YASHCHIKI_PROXY_HTTPS = "http://proxy.kip.uni-heidelberg.de:8080"
- YASHCHIKI_BUILD_SPACK_GCC = 1
- YASHCHIKI_SPACK_GCC_VERSION = "11.2.0"
- YASHCHIKI_SPACK_GCC = "gcc@${YASHCHIKI_SPACK_GCC_VERSION}"
- TMPDIR = "/tmp/${env.NODE_NAME}"
- JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
- returnStdout: true).trim()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
stages {
@@ -60,7 +51,6 @@ pipeline {
script {
cleanupSteps()
}
- sh "mkdir -p \"${env.JOB_TMP_SPACK}\" && chmod 777 \"${env.JOB_TMP_SPACK}\""
}
}
stage('yashchiki Checkout') {
@@ -77,59 +67,49 @@ pipeline {
}
}
}
- stage('Validate environment') {
- steps {
- sh "bash lib/yashchiki/validate_environment.sh"
- }
- }
stage('Dump Meta Info') {
steps {
+ sh "mkdir -p ${YASHCHIKI_META_DIR}"
sh "bash bin/yashchiki_dump_meta_info.sh"
sh "bash bin/yashchiki_notify_gerrit.sh -m 'Build containing this change started..'"
}
}
- stage('Spack Fetch') {
- steps {
- script {
- try {
- sh "bash lib/yashchiki/fetch.sh"
- }
- catch (Throwable t) {
- archiveArtifacts "errors_concretization.log"
- throw t
- }
- spec_folder_in_container = sh(script: "bash lib/yashchiki/get_host_env.sh SPEC_FOLDER_IN_CONTAINER", returnStdout: true).trim()
- archiveArtifacts(artifacts: "sandboxes/*/$spec_folder_in_container/*.yaml", allowEmptyArchive: true)
- }
- }
- }
stage('Deploy utilities') {
steps {
sh "bash bin/yashchiki_deploy_utilities.sh"
}
}
- stage('Create visionary recipe') {
- steps {
- sh "bash share/yashchiki/styles/visionary/create_recipe.sh"
- }
- }
- stage('Build sandbox') {
- steps {
- sh "bash lib/yashchiki/build_sandbox.sh"
- }
- }
stage('Build container image') {
steps {
- sh "bash lib/yashchiki/build_image.sh"
+ script {
+ try {
+ sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_visionary_temp.img " +
+ "--log-dir=log " +
+ "--proxy-http=${YASHCHIKI_PROXY_HTTP} " +
+ "--proxy-https=${YASHCHIKI_PROXY_HTTPS} " +
+ "--tmp-subdir=${env.NODE_NAME} " +
+ "--meta-dir=${YASHCHIKI_META_DIR} " +
+ "--caches-dir=${YASHCHIKI_CACHES_ROOT} " +
+ "--sandboxes-dir=${YASHCHIKI_SANDBOXES} " +
+ "--host-env-filename=${WORKSPACE}/host.env " +
+ "--build-cache-name=${BUILD_CACHE_NAME} " +
+ ("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
+ "--recipe-filename=${WORKSPACE}/visionary_recipe.def"
+ } catch (Throwable t) {
+ archiveArtifacts "errors_concretization.log"
+ throw t
+ }
+ archiveArtifacts(artifacts: "sandboxes/*/opt/spack_specs/*.yaml", allowEmptyArchive: true)
+ archiveArtifacts(artifacts: "log/*.log", allowEmptyArchive: true)
+ }
}
}
- stage('Update build cache and export container') {
+ stage('Export container') {
steps {
script {
// we only want the container name, tail everything else
CONTAINER_IMAGE = sh(script: "bash bin/yashchiki_deploy_container.sh | tail -n 1", returnStdout: true).trim()
}
- sh "bash lib/yashchiki/update_build_cache.sh -c \"$CONTAINER_IMAGE\""
sh "bash bin/yashchiki_notify_gerrit.sh -t Build -c \"$CONTAINER_IMAGE\""
}
}
diff --git a/.ci/Jenkinsfile_asic b/.ci/Jenkinsfile_asic
index 118a6480..b38dda96 100755
--- a/.ci/Jenkinsfile_asic
+++ b/.ci/Jenkinsfile_asic
@@ -28,41 +28,30 @@ pipeline {
}
environment {
- CONTAINER_STYLE = "asic"
YASHCHIKI_HOST_ENV_PATH = "${WORKSPACE}/host.env"
}
stages {
stage('Container Build') {
- agent { label 'conviz1||conviz2' }
+ // TODO: remove once unused
environment {
- DOCKER_BASE_IMAGE = "centos:7"
- // versions from system packages
- DEPENDENCY_PYTHON = "python@3.8.3"
+ CONTAINER_STYLE = "asic"
YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
- YASHCHIKI_RECIPE_PATH = "${WORKSPACE}/asic_recipe.def"
YASHCHIKI_CACHES_ROOT = "${HOME}"
YASHCHIKI_SPACK_PATH = "${env.WORKSPACE}/spack"
- YASHCHIKI_IMAGE_NAME = "singularity_asic_temp.img"
YASHCHIKI_SANDBOXES = "sandboxes"
YASHCHIKI_PROXY_HTTP = "http://proxy.kip.uni-heidelberg.de:8080"
YASHCHIKI_PROXY_HTTPS = "http://proxy.kip.uni-heidelberg.de:8080"
- YASHCHIKI_BUILD_SPACK_GCC = 0
- YASHCHIKI_SPACK_GCC_VERSION = "4.8.5"
- YASHCHIKI_SPACK_GCC = "gcc@${YASHCHIKI_SPACK_GCC_VERSION}"
- TMPDIR = "/tmp/${env.NODE_NAME}"
- JOB_TMP_SPACK = sh(script: "mkdir -p ${env.TMPDIR} &>/dev/null; mktemp -d ${env.TMPDIR}/spack-XXXXXXXXXX",
- returnStdout: true).trim()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
+ agent { label 'conviz1||conviz2' }
stages {
stage('Pre-build Cleanup') {
steps {
script {
cleanupSteps()
}
- sh "mkdir -p \"${env.JOB_TMP_SPACK}\" && chmod 777 \"${env.JOB_TMP_SPACK}\""
}
}
stage('yashchiki Checkout') {
@@ -79,59 +68,49 @@ pipeline {
}
}
}
- stage('Validate environment') {
+ stage('Deploy utilities') {
steps {
- sh "bash lib/yashchiki/validate_environment.sh"
+ sh "bash bin/yashchiki_deploy_utilities.sh"
}
}
stage('Dump Meta Info') {
steps {
+ sh "mkdir -p ${YASHCHIKI_META_DIR}"
sh "bash bin/yashchiki_dump_meta_info.sh"
sh "bash bin/yashchiki_notify_gerrit.sh -m 'Build containing this change started..'"
}
}
- stage('Spack Fetch') {
+ stage('Build container image') {
steps {
script {
try {
- sh "bash lib/yashchiki/fetch.sh"
- }
- catch (Throwable t) {
+ sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_asic_temp.img " +
+ "--log-dir=log " +
+ "--proxy-${YASHCHIKI_PROXY_HTTP} " +
+ "--proxy-https=${YASHCHIKI_PROXY_HTTPS} " +
+ "--tmp-subdir=${env.NODE_NAME} " +
+ "--meta-dir=${YASHCHIKI_META_DIR} " +
+ "--caches-dir=${YASHCHIKI_CACHES_ROOT} " +
+ "--build-cache-name=${BUILD_CACHE_NAME} " +
+ "--sandboxes-dir=${YASHCHIKI_SANDBOXES} " +
+ "--host-env-filename=${WORKSPACE}/host.env " +
+ ("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
+ "--recipe-filename=${WORKSPACE}/asic_recipe.def "
+ } catch (Throwable t) {
archiveArtifacts "errors_concretization.log"
throw t
}
- spec_folder_in_container = sh(script: "lib/yashchiki/get_host_env.sh SPEC_FOLDER_IN_CONTAINER", returnStdout: true).trim()
- archiveArtifacts(artifacts: "sandboxes/*/$spec_folder_in_container/*.yaml", allowEmptyArchive: true)
+ archiveArtifacts(artifacts: "sandboxes/*/opt/spack_specs/*.yaml", allowEmptyArchive: true)
+ archiveArtifacts(artifacts: "log/*.log", allowEmptyArchive: true)
}
}
}
- stage('Deploy utilities') {
- steps {
- sh "bash bin/yashchiki_deploy_utilities.sh"
- }
- }
- stage('Create asic recipe') {
- steps {
- sh "share/yashchiki/styles/asic/create_recipe.sh"
- }
- }
- stage('Build sandbox') {
- steps {
- sh "bash lib/yashchiki/build_sandbox.sh"
- }
- }
- stage('Build container image') {
- steps {
- sh "bash lib/yashchiki/build_image.sh"
- }
- }
- stage('Update build cache and export container') {
+ stage('Export container') {
steps {
script {
// we only want the container name, tail everything else
CONTAINER_IMAGE = sh(script: "bin/yashchiki_deploy_container.sh | tail -n 1", returnStdout: true).trim()
}
- sh "bash lib/yashchiki/update_build_cache.sh -c \"$CONTAINER_IMAGE\""
sh "bash bin/yashchiki_notify_gerrit.sh -t Build -c \"$CONTAINER_IMAGE\""
}
}
diff --git a/bin/yashchiki b/bin/yashchiki
new file mode 100644
index 00000000..10446bd6
--- /dev/null
+++ b/bin/yashchiki
@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+
+import argparse
+import os
+import pathlib
+import subprocess
+import tempfile
+import textwrap
+from typing import Optional
+
+
+def check_no_globbing(path: Optional[str]) -> None:
+ """
+ Check that no globbing characters are used in the given path.
+
+ :param path: Path to check.
+ :raises ContainsGlobError: If glob cahracters are used in path.
+ """
+ globs = ["*", "?", "[", "]", "$", "{", "}", "|"]
+
+ class ContainsGlobError(RuntimeError):
+ pass
+
+ # note: str(None) is 'None' and is therefore handled correctly
+ if any(glob in str(path) for glob in globs):
+ raise ContainsGlobError(f"Path {path} containing any of "
+ f"{' '.join(globs)} is not supported.")
+
+
+class HelpFormatter(
+ argparse.RawDescriptionHelpFormatter,
+ argparse.ArgumentDefaultsHelpFormatter):
+ """
+ Formatting for argument parser help message generation.
+ """
+
+
+parser = argparse.ArgumentParser(
+ prog="yashchiki",
+ formatter_class=HelpFormatter,
+ description=textwrap.dedent("""\
+ Yashchiki singularity image builder.
+
+ For a successful image build, a style of container to build, a spack
+ installation and a name for the resulting image is required.
+
+ Read: yashchiki builds a container of STYLE with spack in SPACK_DIR
+ to OUTPUT.
+ """))
+
+# mandatory
+parser.add_argument(
+ "style", type=str, choices=["visionary", "asic"],
+ help="Style of container to build.")
+parser.add_argument(
+ "spack_dir", type=pathlib.Path,
+ help="Location of spack to use.")
+parser.add_argument(
+ "output", type=pathlib.Path,
+ help="File name of the resulting container image.")
+# optional but important
+parser.add_argument(
+ "--update-build-cache", action="store_true",
+ help="Update build cache.")
+# optional with persistent default
+parser.add_argument(
+ "--caches-dir", type=pathlib.Path,
+ default=os.path.expanduser("~/.yashchiki/"),
+ help="Location of caches to use.")
+parser.add_argument(
+ "--log-dir", type=pathlib.Path,
+ default=os.path.expanduser("~/.yashchiki/log/"),
+ help="Location of logs to use.")
+parser.add_argument(
+ "--sandboxes-dir", type=pathlib.Path,
+ default=os.path.expanduser("~/.yashchiki/sandboxes"),
+ help="Location of sandboxes for container creation to use.")
+# optional with temporary default
+parser.add_argument(
+ "--meta-dir", type=pathlib.Path,
+ help="Folder where to store meta information to be copied into the "
+ "container. If not provided, a temporary directory is used.")
+parser.add_argument(
+ "--host-env-filename", type=pathlib.Path,
+ help="Location of host environment storage file to use within container "
+ "build. If not provided, a temporary location is used.")
+parser.add_argument(
+ "--tmp-subdir", type=pathlib.Path, default="",
+ help=f"Directory under {tempfile.gettempdir()} which to use as root for "
+ "temporary files to be owned by spack.")
+parser.add_argument(
+ "--recipe-filename", type=pathlib.Path,
+ help=f"Explicit filename for singularity recipe to construct. If not "
+ "provided, a temporary location is used.")
+parser.add_argument(
+ "--build-cache-name", type=str, default="default",
+ help="Name of build cache to use, resides under "
+ "<CACHES_DIR>/build_caches/<BUILD_CACHE_NAME>.")
+
+# optional options
+parser.add_argument(
+ "--proxy-http", type=str,
+ help="HTTP proxy to use when required.")
+parser.add_argument(
+ "--proxy-https", type=str,
+ help="HTTPS proxy to use when required.")
+parser.add_argument(
+ "--debug", action="store_true",
+ help="Enable debug-level logging.")
+
+args = parser.parse_args()
+
+# yashchiki program root directory to use for script location
+root_dir = pathlib.Path(__file__).parent.parent
+
+# check provided paths
+if not args.spack_dir.is_dir():
+ raise NotADirectoryError("spack_dir is required to be a path to an "
+ "existing directory.")
+if (args.meta_dir is not None) and (not args.meta_dir.is_dir()):
+ raise NotADirectoryError("meta-dir is required to be a path to an "
+ "existing directory.")
+
+paths = [
+ args.spack_dir,
+ args.output,
+ args.caches_dir,
+ args.log_dir,
+ args.sandboxes_dir,
+ args.meta_dir,
+ args.host_env_filename,
+ args.tmp_subdir,
+ args.recipe_filename
+]
+# ensure no globbing is performed in the paths for shell scripts to work
+for path_to_check in paths:
+ check_no_globbing(path_to_check)
+
+# collection of environment variables used to configure the shell scripts'
+# behavior
+env = {
+ "DOCKER_BASE_IMAGE": "debian:bullseye" if args.style == "visionary" else "centos:7",
+ # This needs to be here because otherwise the default python
+ # (2.7.18) will pollute the spec and lead to a conflict
+ # can be removed as soon as the explicit preferred version
+ # is dropped
+ "DEPENDENCY_PYTHON": "python@3." + ("8.2" if args.style == "visionary" else "6.8"),
+ "YASHCHIKI_BUILD_SPACK_GCC": "1" if args.style == "visionary" else "0",
+ "YASHCHIKI_SPACK_GCC_VERSION": "11.2.0" if args.style == "visionary" else "4.8.5",
+ "YASHCHIKI_SPACK_GCC": "gcc@11.2.0" if args.style == "visionary" else "gcc@4.8.5",
+ "WORKSPACE": os.getcwd(), # FIXME: should not be required
+ "CONTAINER_STYLE": args.style,
+ "CONTAINER_BUILD_TYPE": "testing", # FIXME: should not be required
+ "YASHCHIKI_DEBUG": str(int(args.debug)),
+ "YASHCHIKI_SANDBOXES": args.sandboxes_dir,
+ "YASHCHIKI_IMAGE_NAME": args.output,
+ "YASHCHIKI_SPACK_PATH": args.spack_dir,
+ "YASHCHIKI_BUILD_CACHE_NAME": args.build_cache_name,
+ "TMPDIR": os.path.join(tempfile.gettempdir(), args.tmp_subdir),
+ "YASHCHIKI_CACHES_ROOT": args.caches_dir,
+} | os.environ
+
+
+# optionally forward http{,s} proxy if argument given
+if args.proxy_http:
+ env = env | {"YASHCHIKI_PROXY_HTTP": args.proxy_http}
+
+if args.proxy_https:
+ env = env | {"YASHCHIKI_PROXY_HTTPS": args.proxy_https}
+
+# create directory for logs
+args.log_dir.mkdir(parents=True, exist_ok=True)
+
+pathlib.Path(env["TMPDIR"]).mkdir(exist_ok=True, parents=True)
+
+
+def run(script: str, env: dict, script_args: list = []):
+ """
+ Execute the given script.
+
+ :param script: Script to execute.
+ :param env: Enviroment to use for execution.
+ :param script_args: Arguments to supply to the script.
+ """
+ stdout = ""
+ try:
+ if args.debug:
+ print(f"executing: {script} {script_args}")
+ out = subprocess.run(
+ ["bash", os.path.join(root_dir, script)] + script_args,
+ env=env, check=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, encoding="utf-8")
+ stdout = out.stdout
+ if args.debug:
+ print(stdout)
+ except subprocess.CalledProcessError as error:
+ stdout = error.stdout
+ print(stdout)
+ with args.log_dir.joinpath(
+ script.replace("/", "_") + ".log").open("w+") as file:
+ file.write(stdout)
+ raise
+ else:
+ with args.log_dir.joinpath(
+ script.replace("/", "_") + ".log").open("w+") as file:
+ file.write(stdout)
+
+
+with tempfile.TemporaryDirectory(prefix="spack-", dir=env["TMPDIR"]) \
+ as temporary_directory_spack, \
+ tempfile.TemporaryDirectory() as temporary_directory:
+ temporary_directory = pathlib.Path(temporary_directory)
+
+ env = env | {"JOB_TMP_SPACK": temporary_directory_spack}
+
+ # singularity recipe filename defaults to temporary file
+ if args.recipe_filename is not None:
+ recipe_filename = args.recipe_filename
+ else:
+ recipe_filename = temporary_directory.joinpath("recipe.def")
+ env = env | {"YASHCHIKI_RECIPE_PATH": recipe_filename}
+
+ # meta data directory defaults to temporary folder
+ if args.meta_dir is not None:
+ meta_dir = args.meta_dir
+ else:
+ meta_dir = temporary_directory.joinpath("meta")
+ env = env | {"YASHCHIKI_META_DIR": meta_dir}
+
+ # host environment storage filename defaults to temporary file
+ if args.host_env_filename is not None:
+ host_env_filename = args.host_env_filename
+ else:
+ host_env_filename = temporary_directory.joinpath("host.env")
+ env = env | {"YASHCHIKI_HOST_ENV_PATH": host_env_filename}
+
+ run("lib/yashchiki/validate_environment.sh", env)
+ run("lib/yashchiki/create_spack_user.sh", env)
+ run("lib/yashchiki/create_caches.sh", env)
+ run("lib/yashchiki/fetch.sh", env)
+ run(str(pathlib.Path("share", "yashchiki", "styles", args.style,
+ "create_recipe.sh")),
+ env)
+ run("lib/yashchiki/build_sandbox.sh", env)
+ run("lib/yashchiki/build_image.sh", env)
+ if args.update_build_cache:
+ run("lib/yashchiki/update_build_cache.sh", env, ["-c", args.output])
+ run("lib/yashchiki/restore_host_user_ownership.sh", env)
diff --git a/bin/yashchiki_dump_meta_info.sh b/bin/yashchiki_dump_meta_info.sh
index 1eb17e90..50eea1f6 100755
--- a/bin/yashchiki_dump_meta_info.sh
+++ b/bin/yashchiki_dump_meta_info.sh
@@ -7,24 +7,26 @@ set -Eeuo pipefail
shopt -s inherit_errexit
ROOT_DIR="$(dirname "$(dirname "$(readlink -m "${BASH_SOURCE[0]}")")")"
-source "${ROOT_DIR}/lib/yashchiki/commons.sh"
+source "${ROOT_DIR}/lib/yashchiki/gerrit.sh"
-mkdir -p "${META_DIR_OUTSIDE}"
+mkdir -p "${YASHCHIKI_META_DIR}"
(
- cd "${YASHCHIKI_INSTALL}"
- git log > "${META_DIR_OUTSIDE}/yashchiki_git.log"
- if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
- gerrit_get_current_change_commits \
- > "${META_DIR_OUTSIDE}/current_changes-yashchiki.dat"
+ if [ -n "${YASHCHIKI_INSTALL:-}" ]; then
+ cd "${YASHCHIKI_INSTALL}"
+ git log > "${YASHCHIKI_META_DIR}/yashchiki_git.log"
+ if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
+ gerrit_get_current_change_commits \
+ > "${YASHCHIKI_META_DIR}/current_changes-yashchiki.dat"
+ fi
fi
)
(
cd ${YASHCHIKI_SPACK_PATH}
- git log > "${META_DIR_OUTSIDE}/spack_git.log"
+ git log > "${YASHCHIKI_META_DIR}/spack_git.log"
if [ "${CONTAINER_BUILD_TYPE}" = "testing" ]; then
gerrit_get_current_change_commits \
- > "${META_DIR_OUTSIDE}/current_changes-spack.dat"
+ > "${YASHCHIKI_META_DIR}/current_changes-spack.dat"
fi
)
diff --git a/lib/yashchiki/build_image.sh b/lib/yashchiki/build_image.sh
index 45cffc6b..c07fc662 100755
--- a/lib/yashchiki/build_image.sh
+++ b/lib/yashchiki/build_image.sh
@@ -14,6 +14,11 @@ TARGET_FOLDER="$(find ${YASHCHIKI_SANDBOXES} -mindepth 1 -maxdepth 1)"
# -> it needs to be bind mounted to the sandbox folder
sudo mount --bind "${YASHCHIKI_SPACK_PATH}" "${TARGET_FOLDER}/opt/spack"
+if test -f "${YASHCHIKI_IMAGE_NAME}"; then
+ echo "Image at ${YASHCHIKI_IMAGE_NAME} exists."
+ exit 1
+fi
+
# TODO: singularity 3.1 produces SIF w/o setuid flags on files, using a newer
# singularity for the image build
#sudo singularity build ${YASHCHIKI_IMAGE_NAME} "${TARGET_FOLDER}"
diff --git a/lib/yashchiki/commons.sh b/lib/yashchiki/commons.sh
index d6144c04..5afdceaf 100755
--- a/lib/yashchiki/commons.sh
+++ b/lib/yashchiki/commons.sh
@@ -122,7 +122,7 @@ export PRESERVED_PACKAGES_INSIDE
export PRESERVED_PACKAGES_OUTSIDE
META_DIR_INSIDE="/opt/meta"
-META_DIR_OUTSIDE="$(get_host_env YASHCHIKI_META_DIR)${META_DIR_INSIDE}"
+META_DIR_OUTSIDE="$(get_host_env YASHCHIKI_META_DIR)"
export META_DIR_INSIDE
export META_DIR_OUTSIDE
diff --git a/lib/yashchiki/create_caches.sh b/lib/yashchiki/create_caches.sh
new file mode 100755
index 00000000..88180700
--- /dev/null
+++ b/lib/yashchiki/create_caches.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+shopt -s inherit_errexit 2>/dev/null || true
+
+if [ ! -d "${YASHCHIKI_CACHES_ROOT}" ]; then
+ mkdir -p "${YASHCHIKI_CACHES_ROOT}"
+fi
+
+if [ ! -d "${YASHCHIKI_CACHES_ROOT}/build_caches" ]; then
+ mkdir -p "${YASHCHIKI_CACHES_ROOT}/build_caches"
+fi
+
+if [ ! -d "${YASHCHIKI_CACHES_ROOT}/download_cache" ]; then
+ mkdir -p "${YASHCHIKI_CACHES_ROOT}/download_cache"
+fi
+
+if [ ! -d "${YASHCHIKI_CACHES_ROOT}/spack_ccache" ]; then
+ mkdir -p "${YASHCHIKI_CACHES_ROOT}/spack_ccache"
+fi
+
+if [ ! -d "${YASHCHIKI_CACHES_ROOT}/preserved_packages" ]; then
+ mkdir -p "${YASHCHIKI_CACHES_ROOT}/preserved_packages"
+fi
+
+# spack requires ccache and preserved packages to be accessible within the container
+sudo chown -R spack:nogroup "${YASHCHIKI_CACHES_ROOT}/spack_ccache"
+sudo chown -R spack:nogroup "${YASHCHIKI_CACHES_ROOT}/preserved_packages"
diff --git a/lib/yashchiki/create_spack_user.sh b/lib/yashchiki/create_spack_user.sh
new file mode 100755
index 00000000..2807fd1b
--- /dev/null
+++ b/lib/yashchiki/create_spack_user.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+shopt -s inherit_errexit 2>/dev/null || true
+
+# we need the spack user outside of the container, create it here if it is not present already
+if [ id spack &>/dev/null ]; then
+ sudo useradd spack --uid 888 --no-create-home --system --shell /bin/bash
+fi
diff --git a/lib/yashchiki/gerrit.sh b/lib/yashchiki/gerrit.sh
new file mode 100755
index 00000000..aeccdb6f
--- /dev/null
+++ b/lib/yashchiki/gerrit.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -euo pipefail
+shopt -s inherit_errexit 2>/dev/null || true
+
+# Get gerrit username
+gerrit_username() {
+ echo "${GERRIT_USERNAME:-hudson}"
+}
+
+# Read the current gerrit config from `.gitreview` into global variables:
+# * gerrit_branch
+# * gerrit_remote
+# * gerrit_host
+# * gerrit_port
+# * gerrit_project
+#
+# Unfortunately, since we cannot return values from function, they have to be
+# global variables.
+gerrit_read_config() {
+ local git_dir
+ git_dir="$(git rev-parse --show-toplevel)"
+ # remote branch
+ gerrit_branch="$(grep "^defaultbranch=" "${git_dir}/.gitreview" | cut -d = -f 2)"
+ gerrit_remote="$(grep "^defaultremote=" "${git_dir}/.gitreview" | cut -d = -f 2)"
+ gerrit_host="$(grep "^host=" "${git_dir}/.gitreview" | cut -d = -f 2)"
+ gerrit_port="$(grep "^port=" "${git_dir}/.gitreview" | cut -d = -f 2)"
+ gerrit_project="$(grep "^project=" "${git_dir}/.gitreview" | cut -d = -f 2)"
+}
+
+# Ensure that the gerrit remote is properly set up in the current git directory.
+gerrit_ensure_setup() {
+ gerrit_read_config
+
+ if ! git remote | grep -q "${gerrit_remote}"; then
+ # ensure git review is set up
+ git remote add "${gerrit_remote}" "ssh://$(gerrit_username)@${gerrit_host}:${gerrit_port}/${gerrit_project}"
+ fi
+ git fetch "${gerrit_remote}" "${gerrit_branch}"
+}
+
+gerrit_filter_current_change_commits() {
+ awk '$1 ~ /^commit$/ { commit=$2 }; $1 ~ /^Change-Id:/ { print commit }'
+}
+
+# Get the current stack of changesets in the current git repo as commit ids.
+gerrit_get_current_change_commits() {
+ gerrit_ensure_setup
+
+ # only provide change-ids that are actually present in gerrit
+ comm -1 -2 \
+ <(git log "${gerrit_remote}/${gerrit_branch}..HEAD" \
+ | gerrit_filter_current_change_commits | sort) \
+ <(git ls-remote "${gerrit_remote}" | awk '$2 ~ /^refs\/changes/ { print $1 }' | sort)
+}
+
+# Convenience method to print the ssh command necessary to connect to gerrit.
+#
+# Note: Make sure the gerrit config was read prior to calling this!
+gerrit_cmd_ssh() {
+ echo -n "ssh -p ${gerrit_port} $(gerrit_username)@${gerrit_host} gerrit"
+}
+
+# Post comment on the given change-id
+#
+# Gerrit host/post will be read from current git repository.
+#
+# Args:
+# -c <change>
+# -m <message>
+gerrit_notify_change() {
+ local change=""
+ local message=""
+ local verified=""
+ local opts OPTIND OPTARG
+ while getopts ":c:m:v:" opts; do
+ case "${opts}" in
+ c) change="${OPTARG}"
+ ;;
+ m) message="${OPTARG}"
+ ;;
+ v) verified="${OPTARG}"
+ ;;
+ *)
+ echo "Invalid argument: ${opts}" >&2
+ return 1
+ ;;
+ esac
+ done
+ shift $(( OPTIND - 1 ))
+
+ if [ -z "${change}" ]; then
+ echo "ERROR: No change to post to given!" >&2
+ return 1
+ fi
+ if [ -z "${message}" ]; then
+ echo "ERROR: No message given!" >&2
+ return 1
+ fi
+
+ gerrit_read_config
+ $(gerrit_cmd_ssh) review --message "\"${message}\"" \
+ "$([ -n "${verified}" ] && echo --verified "${verified}")" \
+ "${change}"
+}
diff --git a/lib/yashchiki/restore_host_user_ownership.sh b/lib/yashchiki/restore_host_user_ownership.sh
new file mode 100755
index 00000000..d3230ddb
--- /dev/null
+++ b/lib/yashchiki/restore_host_user_ownership.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+shopt -s inherit_errexit 2>/dev/null || true
+
+if [ -d "${YASHCHIKI_SPACK_PATH}" ]; then
+ sudo chown -R $(id -un):$(id -gn) "${YASHCHIKI_SPACK_PATH}"
+fi
+
+if [ -d "${JOB_TMP_SPACK}" ]; then
+ sudo chown -R $(id -un):$(id -gn) "${JOB_TMP_SPACK}"
+fi
diff --git a/lib/yashchiki/update_build_cache.sh b/lib/yashchiki/update_build_cache.sh
index bac58999..49ff2458 100755
--- a/lib/yashchiki/update_build_cache.sh
+++ b/lib/yashchiki/update_build_cache.sh
@@ -3,12 +3,6 @@
set -euo pipefail
shopt -s inherit_errexit 2>/dev/null || true
-# only update build cache for stable builds
-if [ "${CONTAINER_BUILD_TYPE:-}" != "stable" ]; then
- echo "Not updating build cache for testing builds." 1>&2
- exit 0
-fi
-
usage() { echo "Usage: ${0} -c <container>" 1>&2; exit 1; }
while getopts ":c:" opts; do
--
GitLab