From 82b7070facaa85528a8a7db652d575a6b1d2e213 Mon Sep 17 00:00:00 2001
From: Viktor Vorobev <vorobev@in.tum.de>
Date: Fri, 8 Jan 2021 16:43:23 +0000
Subject: [PATCH] Merged in NRRPLT-8093-ci-for-ubuntu20 (pull request #23)
[NRRPLT-8093] Ubuntu 20.04 Jenkins CI
Approved-by: Ugo Albanese
Approved-by: Krzysztof Lebioda
---
.ci/bitbucket_api_get.bash | 7 ++
.ci/build.bash | 81 ++++++++++++++++++++
.ci/ci_download_directory.py | 60 +++++++++++++++
Jenkinsfile | 144 +++++++++++++++++++++++++++++++++++
4 files changed, 292 insertions(+)
create mode 100644 .ci/bitbucket_api_get.bash
create mode 100644 .ci/build.bash
create mode 100644 .ci/ci_download_directory.py
create mode 100644 Jenkinsfile
diff --git a/.ci/bitbucket_api_get.bash b/.ci/bitbucket_api_get.bash
new file mode 100644
index 0000000..3ca93ca
--- /dev/null
+++ b/.ci/bitbucket_api_get.bash
@@ -0,0 +1,7 @@
+#!/bin/bash
+REPOSITORY=$1
+FILENAME_XSD=$2
+TOPIC_BRANCH=$3
+
+wget https://api.bitbucket.org/2.0/repositories/hbpneurorobotics/${REPOSITORY}/src/${TOPIC_BRANCH}/${FILENAME_XSD} -O ${FILENAME_XSD} \
+ && echo ${FILENAME_XSD}:${TOPIC_BRANCH}
diff --git a/.ci/build.bash b/.ci/build.bash
new file mode 100644
index 0000000..bf962f7
--- /dev/null
+++ b/.ci/build.bash
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -e
+set -x
+
+whoami
+env | sort
+pwd
+
+# import environment
+if [ -f .ci/env ]; then
+ # add quotes to all vars (but do it once)
+ sudo sed -i -E 's/="*(.*[^"])"*$/="\1"/' .ci/env
+ source '.ci/env'
+fi
+
+if [ -z ${PYTHON_VERSION_MAJOR_MINOR} ]; then
+ export PYTHON_VERSION_MAJOR=$(python -c "import sys; print(sys.version_info.major)")
+ export PYTHON_VERSION_MAJOR_MINOR=$(python -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))")
+fi
+
+export HBP=$WORKSPACE
+cd $WORKSPACE
+export PYTHONPATH=
+source ${USER_SCRIPTS_DIR}/nrp_variables
+# source ${USER_SCRIPTS_DIR}/nrp_aliases
+cd ${BRAIN_SIMULATION_DIR}
+
+
+# Configure build has to be placed before make devinstall
+export VIRTUAL_ENV_PATH=$VIRTUAL_ENV
+export NRP_INSTALL_MODE=dev
+export export PYTHONPATH=$VIRTUAL_ENV_PATH/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages:$PYTHONPATH
+# Concatenate all build requirements, ensure newline in between
+(echo; cat ${HBP}/${EXP_CONTROL_DIR}/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
+(echo; cat ${HBP}/${CLE_DIR}/hbp_nrp_cle/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
+(echo; cat ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
+
+# Checkout config.ini.sample from user-scripts
+cp ${HBP}/${USER_SCRIPTS_DIR}/config_files/CLE/config.ini.sample ${HBP}/${CLE_DIR}/hbp_nrp_cle/hbp_nrp_cle/config.ini
+
+# Obtain schemas
+REPOSITORY=experiments
+[ -z "$(git ls-remote --heads git@bitbucket.org:hbpneurorobotics/${REPOSITORY}.git ${TOPIC_BRANCH})" ] \
+ && CO_BRANCH="${DEFAULT_BRANCH}" \
+ || CO_BRANCH="${TOPIC_BRANCH}"
+bash ./.ci/bitbucket_api_get.bash "${REPOSITORY}" bibi_configuration.xsd "${CO_BRANCH}"
+bash ./.ci/bitbucket_api_get.bash "${REPOSITORY}" ExDConfFile.xsd "${CO_BRANCH}"
+python .ci/ci_download_directory.py https://bitbucket.org/hbpneurorobotics/experiments/src/development/hbp-scxml/
+REPOSITORY=models
+[ -z "$(git ls-remote --heads git@bitbucket.org:hbpneurorobotics/${REPOSITORY}.git ${TOPIC_BRANCH})" ] \
+ && CO_BRANCH="${DEFAULT_BRANCH}" \
+ || CO_BRANCH="${TOPIC_BRANCH}"
+bash ./.ci/bitbucket_api_get.bash "${REPOSITORY}" robot_model_configuration.xsd "${CO_BRANCH}"
+bash ./.ci/bitbucket_api_get.bash "${REPOSITORY}" environment_model_configuration.xsd "${CO_BRANCH}"
+
+# Generate schemas
+pushd $VIRTUAL_ENV_PATH/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages && rm -f hbp-nrp-distributed-nest.egg-link && popd
+make devinstall # Otherwise it can't find pyxbgen
+export pyxb_version=`grep "pyxb" ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/requirements.txt`
+. $VIRTUAL_ENV_PATH/bin/activate \
+ && pyxbgen -u bibi_configuration.xsd -m bibi_api_gen \
+ && pyxbgen -u ExDConfFile.xsd -m exp_conf_api_gen \
+ && pyxbgen -u robot_model_configuration.xsd -m robot_conf_api_gen \
+ && pyxbgen -u environment_model_configuration.xsd -m environment_conf_api_gen
+mv bibi_api_gen.py exp_conf_api_gen.py _sc.py robot_conf_api_gen.py environment_conf_api_gen.py ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/hbp_nrp_commons/generated
+touch ${HBP}/${EXDBACKEND_DIR}/hbp_nrp_commons/hbp_nrp_commons/generated/__init__.py
+deactivate
+
+
+# Run tests
+export IGNORE_LINT='platform_venv|migrations|nest|ci_download_directory'
+# Egg-links have to be removed because make devinstall set them up wrongly
+pushd $VIRTUAL_ENV_PATH/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages \
+ && rm -f hbp-nrp-distributed-nest.egg-link && popd
+. $VIRTUAL_ENV_PATH/bin/activate \
+ && source /opt/ros/noetic/setup.bash \
+ && source $HBP/GazeboRosPackages/devel/setup.bash \
+ && echo "PYTHONPATH $PYTHONPATH" \
+ && make verify_base -i
+
\ No newline at end of file
diff --git a/.ci/ci_download_directory.py b/.ci/ci_download_directory.py
new file mode 100644
index 0000000..4c3b0a8
--- /dev/null
+++ b/.ci/ci_download_directory.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+import os
+import sys
+from urllib.parse import urlparse
+import urllib.request
+
+try:
+ import json
+except:
+ import simplejson as json
+
+
+def open_directory(API_PATH, username, repo, slug, path):
+ directory_url = "%s/%s/%s/src/%s/%s" % (API_PATH, username, repo, slug, path)
+ print(directory_url)
+ json_data_url_handle = urllib.request.urlopen(directory_url)
+
+ if json_data_url_handle.code != 200:
+ print("url %s not found" % directory_url)
+ exit()
+
+ json_directory = json.loads(json_data_url_handle.read())
+ page_exists = True
+
+ while page_exists:
+ for item in json_directory["values"]:
+ print(json.dumps(item, indent=4))
+ if item["type"] == "commit_directory":
+ open_directory(API_PATH, username, repo, slug, item["path"])
+
+ for file in json_directory["values"]:
+ if file["type"] == "commit_file" and file["mimetype"] == "text/xml":
+ try:
+ os.makedirs(os.path.dirname(file["path"]))
+ except OSError:
+ None
+ print("downloading %s" % file["path"])
+ print(file['links']['self']['href'])
+ urllib.request.urlretrieve(file['links']['self']['href'], file['path'])
+ if "next" in json_directory:
+ json_data_url_handle = urllib.request.urlopen(json_directory['next'])
+ if json_data_url_handle.code != 200:
+ print("url %s not found" % directory_url)
+ exit()
+ json_directory = json.loads(json_data_url_handle.read())
+ else:
+ page_exists = False
+
+if (
+ len(sys.argv) != 2 or
+ sys.argv[1].find("https://bitbucket.org/") != 0 or
+ sys.argv[1].find("/src/") == -1
+):
+ print("usage: python download_directory.py.py https://bitbucket.org/ws/repo/src/branch/demo/")
+ exit()
+
+API_PATH = "https://api.bitbucket.org/2.0/repositories"
+null, username, repo, null, slug, path = urlparse(sys.argv[1]).path.split("/", 5)
+print(username, repo, slug, path)
+open_directory(API_PATH, username, repo, slug, path)
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..059c002
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,144 @@
+#!groovy
+// Load shared library at master branch
+// the path to the repo with this library should be specified in Jenkins
+// https://tomd.xyz/jenkins-shared-library/
+// https://www.jenkins.io/doc/book/pipeline/shared-libraries/
+@Library('nrp-shared-libs@master') _
+
+pipeline {
+ environment {
+ USER_SCRIPTS_DIR = "user-scripts"
+ ADMIN_SCRIPTS_DIR = "admin-scripts"
+ GAZEBO_ROS_DIR = "GazeboRosPackages"
+ EXP_CONTROL_DIR = "ExperimentControl"
+ CLE_DIR = "CLE"
+ BRAIN_SIMULATION_DIR = "BrainSimulation"
+ EXDBACKEND_DIR = "ExDBackend"
+ // GIT_CHECKOUT_DIR is a dir of the main project (that was pushed)
+ GIT_CHECKOUT_DIR = "${env.BRAIN_SIMULATION_DIR}"
+
+ // selectTopicBranch function is used to choose the correct branch name as topic
+ // the function is defined in shared libs
+ //
+ // In case there is a PR for a branch, then Jenkins runs a pipeline for this pull request, not for the branch,
+ // even if there are new commits to the branch (until PR is not closed). The BRANCH_NAME variable in this case is something like PR-###
+ // The name of the branch which is merged is stored in CHANGE_BRANCH variable. Thus, we should choose CHANGE_BRANCH as topic
+ //
+ // If there is a branch without PR, then Jenkins creates build for it normally for every push and the branch name is stored in BRANCH_NAME variable.
+ // CHANGE_BRANCH is empty in this case. Thus, we choose BRANCH_NAME as topic for branches without PR.
+ TOPIC_BRANCH = selectTopicBranch(env.BRANCH_NAME, env.CHANGE_BRANCH)
+ DEFAULT_BRANCH = 'development'
+
+ NRP_COVERAGE_BRANCH=0
+ NRP_COVERAGE_LINE=23
+ }
+ agent {
+ docker {
+ // NEXUS_REGISTRY_IP and NEXUS_REGISTRY_PORT are Jenkins global variables
+ image "${env.NEXUS_REGISTRY_IP}:${env.NEXUS_REGISTRY_PORT}/nrp:development"
+ args '--entrypoint="" -u root --privileged'
+ }
+ }
+ options {
+ // Skip code checkout prior running pipeline (only Jenkinsfile is checked out)
+ skipDefaultCheckout true
+ }
+
+ stages {
+ stage('Code checkout') {
+ steps {
+ // clear workspace
+ sh "rm -rf *"
+ // Notify BitBucket on the start of the job
+ // The Bitbucket Build Status Notifier is used
+ // REF: https://plugins.jenkins.io/bitbucket-build-status-notifier/
+
+ bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Code checkout')
+
+ // Debug information on available environment
+ echo sh(script: 'env|sort', returnStdout: true)
+
+ // Checkout main project to GIT_CHECKOUT_DIR
+ dir(env.GIT_CHECKOUT_DIR) {
+ checkout scm
+ sh 'chown -R "${USER}" ./'
+ }
+
+ // Clone all dependencies
+ // cloneRepoTopic:
+ // 1 - directory to checkout
+ // 2 - repo
+ // 3 - name of topic branch
+ // 4 - default branch if topic unavailable
+ // 5 - username for chown
+
+ // TODO: fix default branch to master when admin-scripts CI is merged
+ cloneRepoTopic(env.ADMIN_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/admin-scripts.git', env.TOPIC_BRANCH, 'master', '${USER}')
+ cloneRepoTopic(env.USER_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/user-scripts.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
+ cloneRepoTopic(env.GAZEBO_ROS_DIR, 'git@bitbucket.org:hbpneurorobotics/gazeborospackages.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
+ cloneRepoTopic(env.EXP_CONTROL_DIR, 'git@bitbucket.org:hbpneurorobotics/experimentcontrol.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
+ cloneRepoTopic(env.CLE_DIR, 'git@bitbucket.org:hbpneurorobotics/cle.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
+ cloneRepoTopic(env.EXDBACKEND_DIR, 'git@bitbucket.org:hbpneurorobotics/exdbackend.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH, '${USER}')
+
+ }
+ }
+
+ stage('Build GazeboRosPackages') {
+ steps {
+ bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building GazeboRosPackages')
+
+ // Use GazeboRosPackages build script
+ dir(env.GAZEBO_ROS_DIR){
+ // Determine explicitly the shell as bash (needed for proper user-scripts operation)
+ sh 'bash .ci/build.bash ${WORKSPACE}/${USER_SCRIPTS_DIR}/'
+ }
+
+ }
+ }
+ stage('Build and test BrainSimulation') {
+ steps {
+ bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Build and test ' + env.GIT_CHECKOUT_DIR)
+ dir(env.GIT_CHECKOUT_DIR){
+ // this is a workaround to pass all env vars into script run by the other user (now we are root)
+ // STAGE_NAME and CHANGE_* have spaces and is not imported then normally, thus delete it, because it's unneeded
+ // enquote all vars to avoid problems with spaces
+ sh 'env > .ci/env'
+ sh 'sudo -H -u bbpnrsoa bash .ci/build.bash'
+
+ // deliver artifacts
+ step([$class: 'CoberturaPublisher',
+ autoUpdateHealth: true,
+ autoUpdateStability: false,
+ coberturaReportFile: 'coverage.xml',
+ failUnhealthy: true,
+ failUnstable: false,
+ maxNumberOfBuilds: 0,
+ onlyStable: false,
+ sourceEncoding: 'ASCII',
+ zoomCoverageChart: false,
+ lineCoverageTargets: "90.0, 0, 0"])
+ archiveArtifacts 'coverage.xml'
+ }
+ }
+ }
+ }
+
+ post {
+ always {
+ dir(env.GIT_CHECKOUT_DIR){
+ archiveArtifacts 'p*.*'
+ archiveArtifacts 'test-reports/*.*'
+ junit 'test-reports/*.xml'
+ }
+ }
+ aborted {
+ bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build aborted!')
+ }
+ failure {
+ bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build failed, see console output!')
+ }
+ success{
+ bitbucketStatusNotify(buildState: 'SUCCESSFUL', buildDescription: 'branch ' + env.GIT_BRANCH)
+ }
+ }
+}
--
GitLab