Skip to content
Snippets Groups Projects
Commit 96ba554c authored by Viktor Vorobev's avatar Viktor Vorobev
Browse files

Merged in NRRPLT-8630-move-jenkins-pipeline (pull request #36) 

NRRPLT-8630 move jenkins pipeline

Approved-by: Ugo Albanese
Approved-by: Vahid Zolfaghari
parent 57d23d6d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.ci/build.bash
+ 31
− 29
View file @ 96ba554c
......@@ -7,52 +7,54 @@ whoami
env | sort
pwd
# import environment
if [ -f .ci/env ]; then
# add quotes to all vars (but do it once)
sudo sed -i -E 's/="*(.*[^"])"*$/="\1"/' .ci/env
source '.ci/env'
if [ -z "${HBP}" ]; then
echo "USAGE: The HBP variable not specified"
exit 1
fi
if [ -z ${PYTHON_VERSION_MAJOR_MINOR} ]; then
export PYTHON_VERSION_MAJOR=$(python -c "import sys; print(sys.version_info.major)")
export PYTHON_VERSION_MAJOR_MINOR=$(python -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))")
fi
USER_SCRIPTS_DIR=${USER_SCRIPTS_DIR:-"user-scripts"}
EXP_CONTROL_DIR=${EXP_CONTROL_DIR:-"ExperimentControl"}
CLE_DIR=${CLE_DIR:-"CLE"}
BRAIN_SIMULATION_DIR=${BRAIN_SIMULATION_DIR:-"BrainSimulation"}
EXDBACKEND_DIR=${EXDBACKEND_DIR:-"ExDBackend"}
export HBP=$WORKSPACE
cd $WORKSPACE
cd "${HBP}"
export PYTHONPATH=
source ${USER_SCRIPTS_DIR}/nrp_variables
# source ${USER_SCRIPTS_DIR}/nrp_aliases
cd ${BRAIN_SIMULATION_DIR}
source "${HBP}/${USER_SCRIPTS_DIR}/nrp_variables"
cd "${HBP}/${BRAIN_SIMULATION_DIR}"
# Configure build has to be placed before make devinstall
export VIRTUAL_ENV_PATH=$VIRTUAL_ENV
export NRP_INSTALL_MODE=dev
export export PYTHONPATH=$VIRTUAL_ENV_PATH/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages:$PYTHONPATH
# Concatenate all build requirements, ensure newline in between
(echo; cat ${HBP}/${EXP_CONTROL_DIR}/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
(echo; cat ${HBP}/${CLE_DIR}/hbp_nrp_cle/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
(echo; cat ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
(echo; cat "${HBP}/${EXP_CONTROL_DIR}/hbp_nrp_excontrol/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt
(echo; cat "${HBP}/${CLE_DIR}/hbp_nrp_cle/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt
(echo; cat "${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt
# Checkout config.ini.sample from user-scripts
cp ${HBP}/${USER_SCRIPTS_DIR}/config_files/CLE/config.ini.sample ${HBP}/${CLE_DIR}/hbp_nrp_cle/hbp_nrp_cle/config.ini
cp "${HBP}/${USER_SCRIPTS_DIR}/config_files/CLE/config.ini.sample" "${HBP}/${CLE_DIR}/hbp_nrp_cle/hbp_nrp_cle/config.ini"
# Obtain schemas
# This variable is needed for common_makefile during schemas generation
export DEFAULT_CO_BRANCH=${DEFAULT_BRANCH}
export DEFAULT_CO_BRANCH="${DEFAULT_BRANCH}"
export TOPIC_BRANCH
cd ${HBP}/${EXDBACKEND_DIR}
make schemas-install
cd ${HBP}/${BRAIN_SIMULATION_DIR}
cd "${HBP}"
# Run tests
export IGNORE_LINT='platform_venv|migrations|nest|ci_download_directory'
# verify_base-ci fails on dependencies mismach, but ignores linter errors, that are cought by Jenkins afterwards
. $VIRTUAL_ENV_PATH/bin/activate \
&& source /opt/ros/noetic/setup.bash \
&& source $HBP/GazeboRosPackages/devel/setup.bash \
&& echo "PYTHONPATH $PYTHONPATH" \
&& make verify_base-ci
# build CLE and ExDBackend before
# verify_base-ci fails on dependencies mismatch, but ignores linter errors, which are cought by Jenkins afterwards
rm -rf build_env
virtualenv build_env \
&& . build_env/bin/activate \
&& cd "${HBP}/${CLE_DIR}" \
&& make --always-make devinstall \
&& cd "${HBP}/${EXDBACKEND_DIR}" \
&& make --always-make devinstall schemas-install \
&& cd "${HBP}/${BRAIN_SIMULATION_DIR}" \
&& make --always-make verify_base-ci
\ No newline at end of file
Jenkinsfile
+ 1
− 144
View file @ 96ba554c
......@@ -5,147 +5,4 @@
// https://www.jenkins.io/doc/book/pipeline/shared-libraries/
@Library('nrp-shared-libs@master') _
// Before starting pipeline, we try to get the proper image tag
def DEFAULT_BRANCH = 'development'
// selectTopicBranch function is used to choose the correct branch name as topic
// the function is defined in shared libs
//
// In case there is a PR for a branch, then Jenkins runs a pipeline for this pull request, not for the branch,
// even if there are new commits to the branch (until PR is not closed). The BRANCH_NAME variable in this case is something like PR-###
// The name of the branch which is merged is stored in CHANGE_BRANCH variable. Thus, we should choose CHANGE_BRANCH as topic
//
// If there is a branch without PR, then Jenkins creates build for it normally for every push and the branch name is stored in BRANCH_NAME variable.
// CHANGE_BRANCH is empty in this case. Thus, we choose BRANCH_NAME as topic for branches without PR.
def TOPIC_BRANCH = selectTopicBranch(env.BRANCH_NAME, env.CHANGE_BRANCH)
// We try to pull the image with the topic name, or use default tag otherwise
def IMG_TAG = checkImageTag("${TOPIC_BRANCH}", "${DEFAULT_BRANCH}")
pipeline {
environment {
USER_SCRIPTS_DIR = "user-scripts"
ADMIN_SCRIPTS_DIR = "admin-scripts"
GAZEBO_ROS_DIR = "GazeboRosPackages"
EXP_CONTROL_DIR = "ExperimentControl"
CLE_DIR = "CLE"
BRAIN_SIMULATION_DIR = "BrainSimulation"
EXDBACKEND_DIR = "ExDBackend"
// GIT_CHECKOUT_DIR is a dir of the main project (that was pushed)
GIT_CHECKOUT_DIR = "${env.BRAIN_SIMULATION_DIR}"
// That is needed to pass the variables into environment with the same name from
// Jenkins global scope (def ..=..)
TOPIC_BRANCH = "${TOPIC_BRANCH}"
DEFAULT_BRANCH = "${DEFAULT_BRANCH}"
CODE_COVERAGE_LINE = 29
}
agent {
docker {
label 'ci_label'
alwaysPull true
// NEXUS_REGISTRY_IP and NEXUS_REGISTRY_PORT are Jenkins global variables
registryUrl "https://${env.NEXUS_REGISTRY_IP}:${env.NEXUS_REGISTRY_PORT}"
registryCredentialsId 'nexusadmin'
image "nrp:${IMG_TAG}"
args '--entrypoint="" -u root --privileged'
}
}
options {
// Skip code checkout prior running pipeline (only Jenkinsfile is checked out)
skipDefaultCheckout true
}
stages {
stage('Code checkout') {
steps {
// clear workspace
sh "rm -rf *"
// Notify BitBucket on the start of the job
// The Bitbucket Build Status Notifier is used
// REF: https://plugins.jenkins.io/bitbucket-build-status-notifier/
bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Code checkout')
// Debug information on available environment
echo sh(script: 'env|sort', returnStdout: true)
// Checkout main project to GIT_CHECKOUT_DIR
dir(env.GIT_CHECKOUT_DIR) {
checkout([
$class: "GitSCM",
branches: scm.branches,
extensions: [
[$class: 'CloneOption', noTags: false],
[$class: 'LocalBranch', localBranch: "**"]
],
userRemoteConfigs: scm.userRemoteConfigs
])
sh 'chown -R "${USER}" ./'
}
// Clone all dependencies
// cloneRepoTopic:
// 1 - directory to checkout
// 2 - repo
// 3 - name of topic branch
// 4 - default branch if topic unavailable
// 5 - username for chown
cloneRepoTopic(env.ADMIN_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/admin-scripts.git', env.TOPIC_BRANCH, 'master', '${USER}')
cloneRepoTopic(env.USER_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/user-scripts.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
cloneRepoTopic(env.GAZEBO_ROS_DIR, 'git@bitbucket.org:hbpneurorobotics/gazeborospackages.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
cloneRepoTopic(env.EXP_CONTROL_DIR, 'git@bitbucket.org:hbpneurorobotics/experimentcontrol.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
cloneRepoTopic(env.CLE_DIR, 'git@bitbucket.org:hbpneurorobotics/cle.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
cloneRepoTopic(env.EXDBACKEND_DIR, 'git@bitbucket.org:hbpneurorobotics/exdbackend.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
sh "git config --global --add safe.directory '*'"
}
}
stage('Build GazeboRosPackages') {
steps {
bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building GazeboRosPackages')
// Use GazeboRosPackages build script
dir(env.GAZEBO_ROS_DIR){
// Determine explicitly the shell as bash (needed for proper user-scripts operation)
sh 'bash .ci/build.bash ${WORKSPACE}/${USER_SCRIPTS_DIR}/'
}
}
}
stage('Build and test BrainSimulation') {
steps {
bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Build and test ' + env.GIT_CHECKOUT_DIR)
dir(env.GIT_CHECKOUT_DIR){
// this is a workaround to pass all env vars into script run by the other user (now we are root)
// STAGE_NAME and CHANGE_* have spaces and is not imported then normally, thus delete it, because it's unneeded
// enquote all vars to avoid problems with spaces
sh 'env > .ci/env'
sh 'sudo -H -u bbpnrsoa bash .ci/build.bash'
// deliver artifacts
makeReports(false, env.CODE_COVERAGE_LINE)
}
}
}
}
post {
always {
dir(env.GIT_CHECKOUT_DIR){
archiveArtifacts 'p*.*'
archiveArtifacts 'test-reports/*.*'
archiveArtifacts 'coverage.xml'
}
}
aborted {
bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build aborted!')
}
failure {
bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build failed, see console output!')
}
success{
bitbucketStatusNotify(buildState: 'SUCCESSFUL', buildDescription: 'branch ' + env.GIT_BRANCH)
}
}
}
jenkinsCIpipelineBackend("BrainSimulation")
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment