diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 8e21278f2fe2562e5f94a784bc9b812724038b4f..be188bca4dbf49affed95268f7ba23e67ce01741 100755
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -13,6 +13,44 @@ Closure cleanupSteps = {
deleteDirs: true)
}
+Map<String, Object> verificationStages = [
+ visionary: [
+ // BSS1
+ [name: "NMPM Software",
+ job: "bld_gerrit-meta-nmpm-software",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ // BSS2
+ [name: "PPU Toolchain",
+ job: "bld_gerrit-ppu-toolchain-dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ [name: "haldls",
+ job: "bld_gerrit-haldls-dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ [name: "hxtorch",
+ job: "bld_gerrit-hxtorch-dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ [name: "pynn-brainscales",
+ job: "bld_gerrit-pynn-brainscales-dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ [name: "documentation-brainscales2",
+ job: "doc_gerrit_documentation-brainscales2-dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ // Visionary KiCad library
+ [name: "pcb-kicad-lib",
+ job: "bld_gerrit_pcb-kicad-lib_dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
+ // Visionary lab tools
+ [name: "labcontrol",
+ job: "bld_gerrit_labcontrol_dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"]
+ ],
+ asic: [
+ [name: "tools-xilinx",
+ job: "bld_gerrit_tools-xilinx_dependencies",
+ container_image_parameter: "OVERWRITE_DEFAULT_ASIC_CONTAINER_IMAGE"]
+ ]
+]
+
pipeline {
agent { label 'conviz1||conviz2' }
@@ -21,17 +59,11 @@ pipeline {
skipDefaultCheckout()
}
- parameters {
- string(name: 'BUILD_CACHE_NAME',
- defaultValue: 'init_from_2023-10-24',
- description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
- }
-
stages {
stage('Container Build') {
// TODO: remove once unused
environment {
- CONTAINER_STYLE = "visionary"
+ CONTAINER_STYLE = "${params.CONTAINER_STYLE}"
YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
YASHCHIKI_CACHES_ROOT = "${HOME}"
@@ -92,7 +124,7 @@ pipeline {
env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
}
- sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_visionary_temp.img " +
+ sh "python3 bin/yashchiki ${CONTAINER_STYLE} ${WORKSPACE}/spack singularity_visionary_temp.img " +
"--log-dir=log " +
"--tmp-subdir=${env.NODE_NAME} " +
"--meta-dir=${YASHCHIKI_META_DIR} " +
@@ -137,68 +169,23 @@ pipeline {
}
cleanup {
archiveArtifacts "host.env"
- archiveArtifacts "out_singularity_build_visionary_recipe.txt"
+ archiveArtifacts "out_singularity_build_recipe.txt"
}
}
}
// Container verification stage: Build visionary metaprojects
stage('Container Verification') {
- parallel {
-
- // BSS1
- stage('NMPM Software') {
- steps {
- build(job: 'bld_gerrit-meta-nmpm-software',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
-
- // BSS2
- stage('PPU Toolchain') {
- steps {
- build(job: 'bld_gerrit-ppu-toolchain-dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
- stage('haldls') {
- steps {
- build(job: 'bld_gerrit-haldls-dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
- stage('hxtorch') {
- steps {
- build(job: 'bld_gerrit-hxtorch-dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
- stage('pynn-brainscales') {
- steps {
- build(job: 'bld_gerrit-pynn-brainscales-dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
- stage('documentation-brainscales2') {
- steps {
- build(job: 'doc_gerrit_documentation-brainscales2-dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
-
- // Visionary KiCad library
- stage('pcb-kicad-lib') {
- steps {
- build(job: 'bld_gerrit_pcb-kicad-lib_dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
- }
- }
- // Visionary lab tools
- stage('labcontrol') {
- steps {
- build(job: 'bld_gerrit_labcontrol_dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
+ steps {
+ script {
+ def branches = [:]
+ verificationStages[params.CONTAINER_STYLE].each {
+ project -> branches[project.name] = {
+ build(job: project.job,
+ parameters: [string(name: project.container_image_parameter, value: CONTAINER_IMAGE)])
+ }
}
+ parallel branches
}
}
post {
diff --git a/.ci/Jenkinsfile_asic b/.ci/Jenkinsfile_asic
deleted file mode 100755
index fe33da18091bc51ad576fabf8df130f616a81690..0000000000000000000000000000000000000000
--- a/.ci/Jenkinsfile_asic
+++ /dev/null
@@ -1,173 +0,0 @@
-@Library("jenlib") _
-
-Closure cleanupSteps = {
- // NOTE: sudo commands have been manually permitted
- // remove sandboxes
- sh "sudo /bin/rm -rf \"${WORKSPACE}/sandboxes/\" || exit 0"
- // remove tmp (not only for spack)
- sh "sudo /bin/rm -rf \"/tmp/${NODE_NAME}/\""
- // the spack repository gets bind mounted into the sandbox and owned by
- // spack user during build -> revert prior to cleaning worksapce
- sh "[ -d \"$WORKSPACE/spack\" ] && sudo chown -R vis_jenkins \"$WORKSPACE/spack\" || true"
- cleanWs(patterns: [[pattern: 'download_cache/', type: 'EXCLUDE']],
- deleteDirs: true)
-}
-
-pipeline {
- agent none
-
- options {
- timestamps()
- skipDefaultCheckout()
- }
-
- parameters {
- string(name: 'BUILD_CACHE_NAME',
- defaultValue: 'asic_init_from_2021-06-18_1',
- description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
- }
-
- stages {
- stage('Container Build') {
- // TODO: remove once unused
- environment {
- CONTAINER_STYLE = "asic"
- YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
- YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
- YASHCHIKI_CACHES_ROOT = "${HOME}"
- YASHCHIKI_SPACK_PATH = "${env.WORKSPACE}/spack"
- YASHCHIKI_SANDBOXES = "sandboxes"
- YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME = get_build_cache_on_failure_name()
- BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
- }
- agent { label 'conviz1||conviz2' }
- stages {
- stage('Pre-build Cleanup') {
- steps {
- script {
- cleanupSteps()
- }
- }
- }
- stage('yashchiki Checkout') {
- steps {
- script {
- sh "git clone ssh://hudson@brainscales-r.kip.uni-heidelberg.de:29418/waf.git symwaf2ic"
- sh "cd symwaf2ic && singularity exec /containers/stable/latest make"
- if (!env.GERRIT_CHANGE_NUMBER) {
- sh "singularity exec /containers/stable/latest ./symwaf2ic/waf setup --project=yashchiki --clone-depth=2"
- } else {
- sh "singularity exec /containers/stable/latest ./symwaf2ic/waf setup --project=yashchiki --clone-depth=2 --gerrit-changes=${GERRIT_CHANGE_NUMBER} --gerrit-url=ssh://hudson@${GERRIT_HOST}:${GERRIT_PORT}"
- }
- sh "singularity exec /containers/stable/latest ./symwaf2ic/waf configure install"
- }
- }
- }
- stage('Deploy utilities') {
- steps {
- sh "bash bin/yashchiki_deploy_utilities.sh"
- }
- }
- stage('Dump Meta Info') {
- steps {
- sh "mkdir -p ${YASHCHIKI_META_DIR}"
- sh "bash bin/yashchiki_dump_meta_info.sh"
- script {
- if (isTriggeredByGerrit()) {
- sh "bash bin/yashchiki_notify_gerrit.sh -m 'Build containing this change started..'"
- }
- }
- }
- }
- stage('Build container image') {
- steps {
- script {
- try {
- // extract options from gerrit comment
- boolean with_debug = false
- boolean with_spack_verbose = false
- if (isTriggeredByGerrit()) {
- gerrit_comment = jesh(script: "echo '${GERRIT_EVENT_COMMENT_TEXT}' | base64 -d", returnStdout: true)
- with_debug = gerrit_comment.contains("WITH_DEBUG")
- with_spack_verbose = gerrit_comment.contains("WITH_SPACK_VERBOSE")
- env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
- }
-
- sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_asic_temp.img " +
- "--log-dir=log " +
- "--tmp-subdir=${env.NODE_NAME} " +
- "--meta-dir=${YASHCHIKI_META_DIR} " +
- "--caches-dir=${YASHCHIKI_CACHES_ROOT} " +
- "--build-cache-name=${BUILD_CACHE_NAME} " +
- "--sandboxes-dir=${YASHCHIKI_SANDBOXES} " +
- "--host-env-filename=${WORKSPACE}/host.env " +
- ("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
- "--recipe-filename=${WORKSPACE}/asic_recipe.def "
- "--build-cache-on-failure-name=${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME} " +
- (with_debug ? "--debug " : "") +
- (with_spack_verbose ? "--spack-verbose " : "")
- } catch (Throwable t) {
- archiveArtifacts "errors_concretization.log"
- throw t
- }
- archiveArtifacts(artifacts: "sandboxes/*/opt/spack_specs/*.yaml", allowEmptyArchive: true)
- archiveArtifacts(artifacts: "log/*.log", allowEmptyArchive: true)
- }
- }
- }
- stage('Export container') {
- steps {
- script {
- // we only want the container name, tail everything else
- CONTAINER_IMAGE = sh(script: "bin/yashchiki_deploy_container.sh | tail -n 1", returnStdout: true).trim()
- if (isTriggeredByGerrit()) {
- sh "bash bin/yashchiki_notify_gerrit.sh -t Build -c \"$CONTAINER_IMAGE\""
- }
- }
- }
- }
- }
- post {
- failure {
- script {
- cache_failed = sh(script: "bin/yashchiki_create_temporary_build_cache_after_failure.sh", returnStdout: true).trim()
- if (isTriggeredByGerrit()) {
- sh "bash bin/yashchiki_notify_gerrit.sh -v -1 -t Build -m \"Successfully built packages stored in cache. Resume by issuing:\nWITH_CACHE_NAME=${cache_failed}\n\nIn your next gerrit comment, NOT commit message!\""
- }
- }
- }
- cleanup {
- archiveArtifacts "host.env"
- archiveArtifacts "out_singularity_build_asic_recipe.txt"
- // Clean build artifacts because otherwise the latest build from each jenkins job can take up to 50GB.
- // 2 executors and 5 Jenkins-Jobs (testing, testing-spack, testing-asic, stable, stable-asic) will slowly but surely eat away memory.
- script {
- cleanupSteps()
- }
- }
- }
- }
-
- // Container verification stage: Build visionary metaprojects
- stage('Container Verification') {
- parallel {
- stage('tools-xilinx') {
- steps {
- build(job: 'bld_gerrit_tools-xilinx_dependencies',
- parameters: [string(name: 'OVERWRITE_DEFAULT_ASIC_CONTAINER_IMAGE',
- value: CONTAINER_IMAGE)])
- }
- }
- }
- }
- }
- post {
- failure {
- notifyFailure(mattermostChannel: "#spack")
- }
- }
-}
-
-String get_build_cache_on_failure_name() {
- return (CONTAINER_BUILD_TYPE == "testing" ? "c${GERRIT_CHANGE_NUMBER}p${GERRIT_PATCHSET_NUMBER}" : jesh("echo 'stable_\$(date --iso)'", returnStdout: true))
-}
diff --git a/lib/yashchiki/build_sandbox.sh b/lib/yashchiki/build_sandbox.sh
index b4b1df7150966efefcb30f7dd1c0f8f532d583f1..cb80daeccc5a79cfb3cea24d661ac94e75614075 100755
--- a/lib/yashchiki/build_sandbox.sh
+++ b/lib/yashchiki/build_sandbox.sh
@@ -19,4 +19,4 @@ TARGET_FOLDER="${YASHCHIKI_SANDBOXES}/${CONTAINER_STYLE}"
mkdir ${YASHCHIKI_SANDBOXES}
# Do not change: special sudo permit for the host user...
-sudo -E singularity build --sandbox "${TARGET_FOLDER}" "${YASHCHIKI_RECIPE_PATH}" | tee out_singularity_build_${CONTAINER_STYLE}_recipe.txt
+sudo -E singularity build --sandbox "${TARGET_FOLDER}" "${YASHCHIKI_RECIPE_PATH}" | tee out_singularity_build_recipe.txt