diff --git a/.ci/build.bash b/.ci/build.bash index 408b4df608059829fcfceebe001aed34aea3b387..a4fe8004b97de8649837e027c173b6a7b9853a53 100644 --- a/.ci/build.bash +++ b/.ci/build.bash @@ -7,52 +7,54 @@ whoami env | sort pwd -# import environment -if [ -f .ci/env ]; then - # add quotes to all vars (but do it once) - sudo sed -i -E 's/="*(.*[^"])"*$/="\1"/' .ci/env - source '.ci/env' +if [ -z "${HBP}" ]; then + echo "USAGE: The HBP variable not specified" + exit 1 fi -if [ -z ${PYTHON_VERSION_MAJOR_MINOR} ]; then - export PYTHON_VERSION_MAJOR=$(python -c "import sys; print(sys.version_info.major)") - export PYTHON_VERSION_MAJOR_MINOR=$(python -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") -fi +USER_SCRIPTS_DIR=${USER_SCRIPTS_DIR:-"user-scripts"} +EXP_CONTROL_DIR=${EXP_CONTROL_DIR:-"ExperimentControl"} +CLE_DIR=${CLE_DIR:-"CLE"} +BRAIN_SIMULATION_DIR=${BRAIN_SIMULATION_DIR:-"BrainSimulation"} +EXDBACKEND_DIR=${EXDBACKEND_DIR:-"ExDBackend"} -export HBP=$WORKSPACE -cd $WORKSPACE +cd "${HBP}" export PYTHONPATH= -source ${USER_SCRIPTS_DIR}/nrp_variables -# source ${USER_SCRIPTS_DIR}/nrp_aliases -cd ${BRAIN_SIMULATION_DIR} + +source "${HBP}/${USER_SCRIPTS_DIR}/nrp_variables" + +cd "${HBP}/${BRAIN_SIMULATION_DIR}" # Configure build has to be placed before make devinstall export VIRTUAL_ENV_PATH=$VIRTUAL_ENV export NRP_INSTALL_MODE=dev -export export PYTHONPATH=$VIRTUAL_ENV_PATH/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages:$PYTHONPATH # Concatenate all build requirements, ensure newline in between -(echo; cat ${HBP}/${EXP_CONTROL_DIR}/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt -(echo; cat ${HBP}/${CLE_DIR}/hbp_nrp_cle/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt -(echo; cat ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt +(echo; cat "${HBP}/${EXP_CONTROL_DIR}/hbp_nrp_excontrol/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt +(echo; cat "${HBP}/${CLE_DIR}/hbp_nrp_cle/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt +(echo; cat "${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt") >> hbp_nrp_distributed_nest/requirements.txt # Checkout config.ini.sample from user-scripts -cp ${HBP}/${USER_SCRIPTS_DIR}/config_files/CLE/config.ini.sample ${HBP}/${CLE_DIR}/hbp_nrp_cle/hbp_nrp_cle/config.ini +cp "${HBP}/${USER_SCRIPTS_DIR}/config_files/CLE/config.ini.sample" "${HBP}/${CLE_DIR}/hbp_nrp_cle/hbp_nrp_cle/config.ini" # Obtain schemas # This variable is needed for common_makefile during schemas generation -export DEFAULT_CO_BRANCH=${DEFAULT_BRANCH} +export DEFAULT_CO_BRANCH="${DEFAULT_BRANCH}" export TOPIC_BRANCH -cd ${HBP}/${EXDBACKEND_DIR} -make schemas-install -cd ${HBP}/${BRAIN_SIMULATION_DIR} +cd "${HBP}" # Run tests export IGNORE_LINT='platform_venv|migrations|nest|ci_download_directory' -# verify_base-ci fails on dependencies mismach, but ignores linter errors, that are cought by Jenkins afterwards -. $VIRTUAL_ENV_PATH/bin/activate \ - && source /opt/ros/noetic/setup.bash \ - && source $HBP/GazeboRosPackages/devel/setup.bash \ - && echo "PYTHONPATH $PYTHONPATH" \ - && make verify_base-ci + +# build CLE and ExDBackend before +# verify_base-ci fails on dependencies mismatch, but ignores linter errors, which are cought by Jenkins afterwards +rm -rf build_env +virtualenv build_env \ + && . build_env/bin/activate \ + && cd "${HBP}/${CLE_DIR}" \ + && make --always-make devinstall \ + && cd "${HBP}/${EXDBACKEND_DIR}" \ + && make --always-make devinstall schemas-install \ + && cd "${HBP}/${BRAIN_SIMULATION_DIR}" \ + && make --always-make verify_base-ci \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index dc683126a694a55c7db6d08bf3b6a8955ea2b65a..96297f879f4788b0c013993e5121fec5458ca0d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,147 +5,4 @@ // https://www.jenkins.io/doc/book/pipeline/shared-libraries/ @Library('nrp-shared-libs@master') _ -// Before starting pipeline, we try to get the proper image tag -def DEFAULT_BRANCH = 'development' -// selectTopicBranch function is used to choose the correct branch name as topic -// the function is defined in shared libs -// -// In case there is a PR for a branch, then Jenkins runs a pipeline for this pull request, not for the branch, -// even if there are new commits to the branch (until PR is not closed). The BRANCH_NAME variable in this case is something like PR-### -// The name of the branch which is merged is stored in CHANGE_BRANCH variable. Thus, we should choose CHANGE_BRANCH as topic -// -// If there is a branch without PR, then Jenkins creates build for it normally for every push and the branch name is stored in BRANCH_NAME variable. -// CHANGE_BRANCH is empty in this case. Thus, we choose BRANCH_NAME as topic for branches without PR. -def TOPIC_BRANCH = selectTopicBranch(env.BRANCH_NAME, env.CHANGE_BRANCH) -// We try to pull the image with the topic name, or use default tag otherwise -def IMG_TAG = checkImageTag("${TOPIC_BRANCH}", "${DEFAULT_BRANCH}") - -pipeline { - environment { - USER_SCRIPTS_DIR = "user-scripts" - ADMIN_SCRIPTS_DIR = "admin-scripts" - GAZEBO_ROS_DIR = "GazeboRosPackages" - EXP_CONTROL_DIR = "ExperimentControl" - CLE_DIR = "CLE" - BRAIN_SIMULATION_DIR = "BrainSimulation" - EXDBACKEND_DIR = "ExDBackend" - // GIT_CHECKOUT_DIR is a dir of the main project (that was pushed) - GIT_CHECKOUT_DIR = "${env.BRAIN_SIMULATION_DIR}" - - // That is needed to pass the variables into environment with the same name from - // Jenkins global scope (def ..=..) - TOPIC_BRANCH = "${TOPIC_BRANCH}" - DEFAULT_BRANCH = "${DEFAULT_BRANCH}" - - CODE_COVERAGE_LINE = 29 - } - agent { - docker { - label 'ci_label' - alwaysPull true - // NEXUS_REGISTRY_IP and NEXUS_REGISTRY_PORT are Jenkins global variables - registryUrl "https://${env.NEXUS_REGISTRY_IP}:${env.NEXUS_REGISTRY_PORT}" - registryCredentialsId 'nexusadmin' - image "nrp:${IMG_TAG}" - args '--entrypoint="" -u root --privileged' - } - } - options { - // Skip code checkout prior running pipeline (only Jenkinsfile is checked out) - skipDefaultCheckout true - } - - stages { - stage('Code checkout') { - steps { - // clear workspace - sh "rm -rf *" - // Notify BitBucket on the start of the job - // The Bitbucket Build Status Notifier is used - // REF: https://plugins.jenkins.io/bitbucket-build-status-notifier/ - - bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Code checkout') - - // Debug information on available environment - echo sh(script: 'env|sort', returnStdout: true) - - // Checkout main project to GIT_CHECKOUT_DIR - dir(env.GIT_CHECKOUT_DIR) { - checkout([ - $class: "GitSCM", - branches: scm.branches, - extensions: [ - [$class: 'CloneOption', noTags: false], - [$class: 'LocalBranch', localBranch: "**"] - ], - userRemoteConfigs: scm.userRemoteConfigs - ]) - sh 'chown -R "${USER}" ./' - } - - // Clone all dependencies - // cloneRepoTopic: - // 1 - directory to checkout - // 2 - repo - // 3 - name of topic branch - // 4 - default branch if topic unavailable - // 5 - username for chown - cloneRepoTopic(env.ADMIN_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/admin-scripts.git', env.TOPIC_BRANCH, 'master', '${USER}') - cloneRepoTopic(env.USER_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/user-scripts.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}') - cloneRepoTopic(env.GAZEBO_ROS_DIR, 'git@bitbucket.org:hbpneurorobotics/gazeborospackages.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}') - cloneRepoTopic(env.EXP_CONTROL_DIR, 'git@bitbucket.org:hbpneurorobotics/experimentcontrol.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}') - cloneRepoTopic(env.CLE_DIR, 'git@bitbucket.org:hbpneurorobotics/cle.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}') - cloneRepoTopic(env.EXDBACKEND_DIR, 'git@bitbucket.org:hbpneurorobotics/exdbackend.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}') - - sh "git config --global --add safe.directory '*'" - } - } - - stage('Build GazeboRosPackages') { - steps { - bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building GazeboRosPackages') - - // Use GazeboRosPackages build script - dir(env.GAZEBO_ROS_DIR){ - // Determine explicitly the shell as bash (needed for proper user-scripts operation) - sh 'bash .ci/build.bash ${WORKSPACE}/${USER_SCRIPTS_DIR}/' - } - - } - } - stage('Build and test BrainSimulation') { - steps { - bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Build and test ' + env.GIT_CHECKOUT_DIR) - dir(env.GIT_CHECKOUT_DIR){ - // this is a workaround to pass all env vars into script run by the other user (now we are root) - // STAGE_NAME and CHANGE_* have spaces and is not imported then normally, thus delete it, because it's unneeded - // enquote all vars to avoid problems with spaces - sh 'env > .ci/env' - sh 'sudo -H -u bbpnrsoa bash .ci/build.bash' - - // deliver artifacts - makeReports(false, env.CODE_COVERAGE_LINE) - } - } - } - } - - post { - always { - dir(env.GIT_CHECKOUT_DIR){ - archiveArtifacts 'p*.*' - archiveArtifacts 'test-reports/*.*' - archiveArtifacts 'coverage.xml' - } - } - aborted { - bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build aborted!') - } - failure { - bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build failed, see console output!') - } - success{ - bitbucketStatusNotify(buildState: 'SUCCESSFUL', buildDescription: 'branch ' + env.GIT_BRANCH) - } - } -} +jenkinsCIpipelineBackend("BrainSimulation")