Skip to content
Snippets Groups Projects
Commit 6fff2c1b authored by Philipp Spilger's avatar Philipp Spilger
Browse files

feat: Use params.CONTAINER_STYLE in Jenkinsfile to merge Jenkinsfile_asic and Jenkinsfile 

* removes code-duplication in Jenkinsfiles

Change-Id: I758e4630286a9da8e52a421493097b3fe951706e
parent 3cde6862
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.ci/Jenkinsfile
+ 50
− 63
View file @ 6fff2c1b
......@@ -13,6 +13,44 @@ Closure cleanupSteps = {
deleteDirs: true)
}
Map<String, Object> verificationStages = [
visionary: [
// BSS1
[name: "NMPM Software",
job: "bld_gerrit-meta-nmpm-software",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
// BSS2
[name: "PPU Toolchain",
job: "bld_gerrit-ppu-toolchain-dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
[name: "haldls",
job: "bld_gerrit-haldls-dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
[name: "hxtorch",
job: "bld_gerrit-hxtorch-dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
[name: "pynn-brainscales",
job: "bld_gerrit-pynn-brainscales-dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
[name: "documentation-brainscales2",
job: "doc_gerrit_documentation-brainscales2-dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
// Visionary KiCad library
[name: "pcb-kicad-lib",
job: "bld_gerrit_pcb-kicad-lib_dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"],
// Visionary lab tools
[name: "labcontrol",
job: "bld_gerrit_labcontrol_dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_CONTAINER_IMAGE"]
],
asic: [
[name: "tools-xilinx",
job: "bld_gerrit_tools-xilinx_dependencies",
container_image_parameter: "OVERWRITE_DEFAULT_ASIC_CONTAINER_IMAGE"]
]
]
pipeline {
agent { label 'conviz1||conviz2' }
......@@ -21,17 +59,11 @@ pipeline {
skipDefaultCheckout()
}
parameters {
string(name: 'BUILD_CACHE_NAME',
defaultValue: 'init_from_2023-10-24',
description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
}
stages {
stage('Container Build') {
// TODO: remove once unused
environment {
CONTAINER_STYLE = "visionary"
CONTAINER_STYLE = "${params.CONTAINER_STYLE}"
YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
YASHCHIKI_CACHES_ROOT = "${HOME}"
......@@ -92,7 +124,7 @@ pipeline {
env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
}
sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_visionary_temp.img " +
sh "python3 bin/yashchiki ${CONTAINER_STYLE} ${WORKSPACE}/spack singularity_visionary_temp.img " +
"--log-dir=log " +
"--tmp-subdir=${env.NODE_NAME} " +
"--meta-dir=${YASHCHIKI_META_DIR} " +
......@@ -137,68 +169,23 @@ pipeline {
}
cleanup {
archiveArtifacts "host.env"
archiveArtifacts "out_singularity_build_visionary_recipe.txt"
archiveArtifacts "out_singularity_build_recipe.txt"
}
}
}
// Container verification stage: Build visionary metaprojects
stage('Container Verification') {
parallel {
// BSS1
stage('NMPM Software') {
steps {
build(job: 'bld_gerrit-meta-nmpm-software',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
// BSS2
stage('PPU Toolchain') {
steps {
build(job: 'bld_gerrit-ppu-toolchain-dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
stage('haldls') {
steps {
build(job: 'bld_gerrit-haldls-dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
stage('hxtorch') {
steps {
build(job: 'bld_gerrit-hxtorch-dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
stage('pynn-brainscales') {
steps {
build(job: 'bld_gerrit-pynn-brainscales-dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
stage('documentation-brainscales2') {
steps {
build(job: 'doc_gerrit_documentation-brainscales2-dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
// Visionary KiCad library
stage('pcb-kicad-lib') {
steps {
build(job: 'bld_gerrit_pcb-kicad-lib_dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
}
}
// Visionary lab tools
stage('labcontrol') {
steps {
build(job: 'bld_gerrit_labcontrol_dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_CONTAINER_IMAGE', value: CONTAINER_IMAGE)])
steps {
script {
def branches = [:]
verificationStages[params.CONTAINER_STYLE].each {
project -> branches[project.name] = {
build(job: project.job,
parameters: [string(name: project.container_image_parameter, value: CONTAINER_IMAGE)])
}
}
parallel branches
}
}
post {
......
.ci/Jenkinsfile_asic deleted 100755 → 0
+ 0
− 173
View file @ 3cde6862
@Library("jenlib") _
Closure cleanupSteps = {
// NOTE: sudo commands have been manually permitted
// remove sandboxes
sh "sudo /bin/rm -rf \"${WORKSPACE}/sandboxes/\" || exit 0"
// remove tmp (not only for spack)
sh "sudo /bin/rm -rf \"/tmp/${NODE_NAME}/\""
// the spack repository gets bind mounted into the sandbox and owned by
// spack user during build -> revert prior to cleaning worksapce
sh "[ -d \"$WORKSPACE/spack\" ] && sudo chown -R vis_jenkins \"$WORKSPACE/spack\" || true"
cleanWs(patterns: [[pattern: 'download_cache/', type: 'EXCLUDE']],
deleteDirs: true)
}
pipeline {
agent none
options {
timestamps()
skipDefaultCheckout()
}
parameters {
string(name: 'BUILD_CACHE_NAME',
defaultValue: 'asic_init_from_2021-06-18_1',
description: 'Which buildcache to use? They reside under $HOME/build_caches/$BUILD_CACHE_NAME and will be created if they do not exist.')
}
stages {
stage('Container Build') {
// TODO: remove once unused
environment {
CONTAINER_STYLE = "asic"
YASHCHIKI_INSTALL = "${WORKSPACE}/yashchiki"
YASHCHIKI_META_DIR = "${WORKSPACE}/meta"
YASHCHIKI_CACHES_ROOT = "${HOME}"
YASHCHIKI_SPACK_PATH = "${env.WORKSPACE}/spack"
YASHCHIKI_SANDBOXES = "sandboxes"
YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME = get_build_cache_on_failure_name()
BUILD_CACHE_NAME = "${params.BUILD_CACHE_NAME}" // propagate parameter to environment
}
agent { label 'conviz1||conviz2' }
stages {
stage('Pre-build Cleanup') {
steps {
script {
cleanupSteps()
}
}
}
stage('yashchiki Checkout') {
steps {
script {
sh "git clone ssh://hudson@brainscales-r.kip.uni-heidelberg.de:29418/waf.git symwaf2ic"
sh "cd symwaf2ic && singularity exec /containers/stable/latest make"
if (!env.GERRIT_CHANGE_NUMBER) {
sh "singularity exec /containers/stable/latest ./symwaf2ic/waf setup --project=yashchiki --clone-depth=2"
} else {
sh "singularity exec /containers/stable/latest ./symwaf2ic/waf setup --project=yashchiki --clone-depth=2 --gerrit-changes=${GERRIT_CHANGE_NUMBER} --gerrit-url=ssh://hudson@${GERRIT_HOST}:${GERRIT_PORT}"
}
sh "singularity exec /containers/stable/latest ./symwaf2ic/waf configure install"
}
}
}
stage('Deploy utilities') {
steps {
sh "bash bin/yashchiki_deploy_utilities.sh"
}
}
stage('Dump Meta Info') {
steps {
sh "mkdir -p ${YASHCHIKI_META_DIR}"
sh "bash bin/yashchiki_dump_meta_info.sh"
script {
if (isTriggeredByGerrit()) {
sh "bash bin/yashchiki_notify_gerrit.sh -m 'Build containing this change started..'"
}
}
}
}
stage('Build container image') {
steps {
script {
try {
// extract options from gerrit comment
boolean with_debug = false
boolean with_spack_verbose = false
if (isTriggeredByGerrit()) {
gerrit_comment = jesh(script: "echo '${GERRIT_EVENT_COMMENT_TEXT}' | base64 -d", returnStdout: true)
with_debug = gerrit_comment.contains("WITH_DEBUG")
with_spack_verbose = gerrit_comment.contains("WITH_SPACK_VERBOSE")
env.BUILD_CACHE_NAME = jesh(script: "bash bin/yashchiki_get_build_cache_name.sh", returnStdout: true)
}
sh "python3 bin/yashchiki visionary ${WORKSPACE}/spack singularity_asic_temp.img " +
"--log-dir=log " +
"--tmp-subdir=${env.NODE_NAME} " +
"--meta-dir=${YASHCHIKI_META_DIR} " +
"--caches-dir=${YASHCHIKI_CACHES_ROOT} " +
"--build-cache-name=${BUILD_CACHE_NAME} " +
"--sandboxes-dir=${YASHCHIKI_SANDBOXES} " +
"--host-env-filename=${WORKSPACE}/host.env " +
("${CONTAINER_BUILD_TYPE}" == "stable" ? "--update-build-cache " : "") +
"--recipe-filename=${WORKSPACE}/asic_recipe.def "
"--build-cache-on-failure-name=${YASHCHIKI_BUILD_CACHE_ON_FAILURE_NAME} " +
(with_debug ? "--debug " : "") +
(with_spack_verbose ? "--spack-verbose " : "")
} catch (Throwable t) {
archiveArtifacts "errors_concretization.log"
throw t
}
archiveArtifacts(artifacts: "sandboxes/*/opt/spack_specs/*.yaml", allowEmptyArchive: true)
archiveArtifacts(artifacts: "log/*.log", allowEmptyArchive: true)
}
}
}
stage('Export container') {
steps {
script {
// we only want the container name, tail everything else
CONTAINER_IMAGE = sh(script: "bin/yashchiki_deploy_container.sh | tail -n 1", returnStdout: true).trim()
if (isTriggeredByGerrit()) {
sh "bash bin/yashchiki_notify_gerrit.sh -t Build -c \"$CONTAINER_IMAGE\""
}
}
}
}
}
post {
failure {
script {
cache_failed = sh(script: "bin/yashchiki_create_temporary_build_cache_after_failure.sh", returnStdout: true).trim()
if (isTriggeredByGerrit()) {
sh "bash bin/yashchiki_notify_gerrit.sh -v -1 -t Build -m \"Successfully built packages stored in cache. Resume by issuing:\nWITH_CACHE_NAME=${cache_failed}\n\nIn your next gerrit comment, NOT commit message!\""
}
}
}
cleanup {
archiveArtifacts "host.env"
archiveArtifacts "out_singularity_build_asic_recipe.txt"
// Clean build artifacts because otherwise the latest build from each jenkins job can take up to 50GB.
// 2 executors and 5 Jenkins-Jobs (testing, testing-spack, testing-asic, stable, stable-asic) will slowly but surely eat away memory.
script {
cleanupSteps()
}
}
}
}
// Container verification stage: Build visionary metaprojects
stage('Container Verification') {
parallel {
stage('tools-xilinx') {
steps {
build(job: 'bld_gerrit_tools-xilinx_dependencies',
parameters: [string(name: 'OVERWRITE_DEFAULT_ASIC_CONTAINER_IMAGE',
value: CONTAINER_IMAGE)])
}
}
}
}
}
post {
failure {
notifyFailure(mattermostChannel: "#spack")
}
}
}
String get_build_cache_on_failure_name() {
return (CONTAINER_BUILD_TYPE == "testing" ? "c${GERRIT_CHANGE_NUMBER}p${GERRIT_PATCHSET_NUMBER}" : jesh("echo 'stable_\$(date --iso)'", returnStdout: true))
}
lib/yashchiki/build_sandbox.sh
+ 1
− 1
View file @ 6fff2c1b
......@@ -19,4 +19,4 @@ TARGET_FOLDER="${YASHCHIKI_SANDBOXES}/${CONTAINER_STYLE}"
mkdir ${YASHCHIKI_SANDBOXES}
# Do not change: special sudo permit for the host user...
sudo -E singularity build --sandbox "${TARGET_FOLDER}" "${YASHCHIKI_RECIPE_PATH}" | tee out_singularity_build_${CONTAINER_STYLE}_recipe.txt
sudo -E singularity build --sandbox "${TARGET_FOLDER}" "${YASHCHIKI_RECIPE_PATH}" | tee out_singularity_build_recipe.txt
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment