.ci/build.bash

@@ -6,10 +6,9 @@ set -x
 whoami
 env | sort
 
-if [ -f .ci/env ]; then
-    # add quotes to all vars (but do it once)
-    sudo sed -i -E 's/="*(.*[^"])"*$/="\1"/' .ci/env 
-    source '.ci/env'
+if [ -z "${HBP}" ]; then
+    echo "USAGE: The HBP variable not specified"
+    exit 1
 fi
 
 if [ -z ${PYTHON_VERSION_MAJOR_MINOR} ]; then
@@ -17,24 +16,23 @@ if [ -z ${PYTHON_VERSION_MAJOR_MINOR} ]; then
     export PYTHON_VERSION_MAJOR_MINOR=$(python -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))")
 fi
 
-# Force the build to happen in the build workspace, not in the container
-# This should be done at the beginning, but there are some circular dependencies
-# between Gazebo build artifacts (existing in the container) and nrp aliases/variables
-
-export HBP=$WORKSPACE
-cd $WORKSPACE
+# Force the build to happen in the build workspace
+cd $HBP
 export PYTHONPATH=
-source ${USER_SCRIPTS_DIR}/nrp_variables
 
-cd ${WORKSPACE}/${VIRTUAL_COACH_DIR}
+VIRTUAL_COACH_DIR=${GIT_CHECKOUT_DIR:-"VirtualCoach"}
+
+cd ${HBP}/${VIRTUAL_COACH_DIR}
 
 
 # Configure build
-export VIRTUAL_ENV_PATH=$VIRTUAL_ENV
 export NRP_INSTALL_MODE=dev
-export PYTHONPATH=hbp_nrp_virtual_coach:${VIRTUAL_ENV_PATH}/lib/python${PYTHON_VERSION_MAJOR_MINOR}/site-packages:$PYTHONPATH
+export PYTHONPATH=hbp_nrp_virtual_coach:$PYTHONPATH
 
 # Build
 export IGNORE_LINT='platform_venv|config_files|examples/integration_test/test_experiment_folder'
-make devinstall
-make verify_base -i
+# verify_base-ci fails on dependencies mismatch, but ignores linter errors, which are cought by Jenkins afterwards
+rm -rf build_env
+virtualenv build_env \
+    && . build_env/bin/activate \
+    && make --always-make verify_base-ci

.ci/whl-package.bash

@@ -2,12 +2,11 @@
 set -e
 
 # Build package
-git fetch --tags
-PYNRP_VERSION=$(git describe --tags --always)
-
-echo "${PYNRP_VERSION}"
-echo "VERSION = \"${PYNRP_VERSION}\"" > hbp_nrp_virtual_coach/pynrp/version.py
+export NRP_INSTALL_MODE=user
+make set-nrp-version
 
 pushd hbp_nrp_virtual_coach
 python3 setup.py sdist bdist_wheel
 popd
+
+cp hbp_nrp_virtual_coach/dist/*.whl ./
\ No newline at end of file

Jenkinsfile

@@ -5,145 +5,4 @@
 // https://www.jenkins.io/doc/book/pipeline/shared-libraries/
 @Library('nrp-shared-libs@master') _
 
-// Before starting pipeline, we try to get the proper image tag
-def DEFAULT_BRANCH = 'development'
-// selectTopicBranch function is used to choose the correct branch name as topic
-// the function is defined in shared libs
-// 
-// In case there is a PR for a branch, then Jenkins runs a pipeline for this pull request, not for the branch, 
-// even if there are new commits to the branch (until PR is not closed). The BRANCH_NAME variable in this case is something like PR-###
-// The name of the branch which is merged is stored in CHANGE_BRANCH variable. Thus, we should choose CHANGE_BRANCH as topic
-//
-// If there is a branch without PR, then Jenkins creates build for it normally for every push and the branch name is stored in BRANCH_NAME variable.
-// CHANGE_BRANCH is empty in this case. Thus, we choose BRANCH_NAME as topic for branches without PR.
-def TOPIC_BRANCH = selectTopicBranch(env.BRANCH_NAME, env.CHANGE_BRANCH)
-// We try to pull the image with the topic name, or use default tag otherwise
-def IMG_TAG = checkImageTag("${TOPIC_BRANCH}", "${DEFAULT_BRANCH}")
-
-pipeline {
-    environment {
-        USER_SCRIPTS_DIR = "user-scripts"
-        ADMIN_SCRIPTS_DIR = "admin-scripts"
-        GAZEBO_ROS_DIR = "GazeboRosPackages"
-        VIRTUAL_COACH_DIR = "VirtualCoach"
-        // GIT_CHECKOUT_DIR is a dir of the main project (that was pushed)
-        GIT_CHECKOUT_DIR = "${env.VIRTUAL_COACH_DIR}"
-
-        // That is needed to pass the variables into environment with the same name from 
-        // Jenkins global scope (def ..=..)
-        TOPIC_BRANCH = "${TOPIC_BRANCH}"
-        DEFAULT_BRANCH = "${DEFAULT_BRANCH}"
-
-        NexusPyPiRepoUrl = "https://nexus.neurorobotics.ebrains.eu/repository/nrp-pypi-internal/"
-
-        CODE_COVERAGE_LINE = 74
-    }
-    agent {
-        docker {
-            label 'ci_label'
-
-            // NEXUS_REGISTRY_IP and NEXUS_REGISTRY_PORT are Jenkins global variables
-            image "${env.NEXUS_REGISTRY_IP}:${env.NEXUS_REGISTRY_PORT}/nrp:${IMG_TAG}"
-            args '--entrypoint="" -u root --privileged'
-        }
-    }
-    options { 
-        // Skip code checkout prior running pipeline (only Jenkinsfile is checked out)
-        skipDefaultCheckout true
-    }
-
-    stages {
-        stage('Code checkout') {
-            steps {
-                // clear workspace
-                sh "rm -rf *"
-                // Notify BitBucket on the start of the job
-                // The Bitbucket Build Status Notifier is used
-                // REF: https://plugins.jenkins.io/bitbucket-build-status-notifier/
-                
-                bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Code checkout')
-
-                // Debug information on available environment
-                echo sh(script: 'env|sort', returnStdout: true)
-
-                // Checkout main project to GIT_CHECKOUT_DIR
-                dir(env.GIT_CHECKOUT_DIR) {
-                    checkout scm
-                    sh 'chown -R "${USER}" ./'
-                }
-
-                // Clone all dependencies
-                cloneRepoTopic(env.USER_SCRIPTS_DIR,  'git@bitbucket.org:hbpneurorobotics/user-scripts.git',      env.TOPIC_BRANCH, env.DEFAULT_BRANCH,     '${USER}')
-                cloneRepoTopic(env.GAZEBO_ROS_DIR,    'git@bitbucket.org:hbpneurorobotics/gazeborospackages.git', env.TOPIC_BRANCH, env.DEFAULT_BRANCH,  '${USER}')
-                cloneRepoTopic(env.ADMIN_SCRIPTS_DIR, 'git@bitbucket.org:hbpneurorobotics/admin-scripts.git',     env.TOPIC_BRANCH, 'master',       '${USER}')
-
-            }
-        }
-
-        stage('Build GazeboRosPackages') {
-            steps {
-                bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building GazeboRosPackages')
-                
-                // Use GazeboRosPackages build script
-                dir(env.GAZEBO_ROS_DIR){
-                    // Determine explicitly the shell as bash (needed for proper user-scripts operation)
-                    sh 'bash .ci/build.bash ${WORKSPACE}/${USER_SCRIPTS_DIR}/'
-                }
-                
-            }
-        }
-
-        stage('Build VirtualCoach') {
-            steps {
-                bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building VirtualCoach')
-
-                // Determine explicitly the shell as bash (needed for proper user-scripts operation)
-                dir(env.GIT_CHECKOUT_DIR){
-                    // this is a workaround to pass all env vars into script run by the other user (now we are root)
-                    sh 'env > .ci/env'
-                    sh 'sudo -H -u bbpnrsoa bash .ci/build.bash'
-
-                    // deliver artifacts
-                    // Make true to false when the coverage reaches 85%-90%. This with turn off auto threshold for coverage
-                    makeReports(true, env.CODE_COVERAGE_LINE)
-                }
-            }
-        }
-
-        stage('Create package') {
-            steps {
-                bitbucketStatusNotify(buildState: 'INPROGRESS', buildName: 'Building pynrp')
-
-                dir(env.GIT_CHECKOUT_DIR){
-                    // tbuild package
-                    sh 'bash ./.ci/whl-package.bash'
-                    // upload package
-                    withCredentials([usernamePassword(credentialsId: 'nexusadmin', usernameVariable: 'USER', passwordVariable: 'PASSWORD')])
-                    {
-                        sh 'pip3 install twine packaging'
-                        sh 'twine upload -u $USER -p $PASSWORD --repository-url ${NexusPyPiRepoUrl} hbp_nrp_virtual_coach/dist/*.whl'
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            dir(env.GIT_CHECKOUT_DIR){
-                archiveArtifacts 'p*.*'
-                archiveArtifacts 'test-reports/*.*'
-                archiveArtifacts 'coverage.xml'
-            }
-        }
-        aborted {
-            bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build aborted!')
-        }
-        failure {
-            bitbucketStatusNotify(buildState: 'FAILED', buildDescription: 'Build failed, see console output!')
-        }
-        success{
-            bitbucketStatusNotify(buildState: 'SUCCESSFUL', buildDescription: 'branch ' + env.GIT_BRANCH)
-        }
-    }
-}
+jenkinsCIpipelinePythonModule("VirtualCoach")

Makefile

@@ -18,7 +18,7 @@ PYTHON_PIP_VERSION?=pip>=19
 python_version_full := $(shell python -c "import sys; maj, min = sys.version_info[:2]; print('{}.{}'.format(maj, min))" 2>&1)
 
 ifeq ($(NRP_INSTALL_MODE),user)
-        include user_makefile
+        include $(HBP)/user-scripts/config_files/user_makefile
 else
         CI_REPO?=git@bitbucket.org:hbpneurorobotics/admin-scripts.git
         CI_DIR?=$(HBP)/admin-scripts/ContinuousIntegration

README.md

 This repository is part of the Neurorobotics Platform software
 Copyright (C) Human Brain Project
-https://neurorobotics.ai
+https://neurorobotics.net
 
 The Human Brain Project is a European Commission funded project
 in the frame of the [Horizon2020 FET Flagship plan](http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships).
 
+This work has received funding from the European Union’s Horizon 2020 Framework Programme for Research and Innovation under the Specific Grant Agreement No. 720270 (Human Brain Project SGA1), and the Specific Grant Agreement No. 785907 (Human Brain Project SGA2), and under the Specific Grant Agreement No. 945539 (Human Brain Project SGA3).
+
 
 You are free to clone this repository and amend its code with respect to
 the license that you find in the root folder.

hbp_nrp_virtual_coach/doc/source/tutorials/launching_exp.rst

@@ -69,11 +69,11 @@ The first thing we need to do is to import the Virtual Coach and create a Virtua
     from pynrp.virtual_coach import VirtualCoach
     vc = VirtualCoach(environment='http://localhost:9000', storage_username='nrpuser', storage_password='password')
 
-Here we chose the local machine as the NRP server. The Virtual Coach can connect to NRP servers running in other locations by setting the *environment* parameter to the proper URL. For example, it is possible to connect to the NRP server running at *http://148.187.149.212* by executing this command:
+Here we chose the local machine as the NRP server. The Virtual Coach can connect to NRP servers running in other locations by setting the *environment* parameter to the proper URL. For example, it is possible to connect to the NRP server running at *https://nrp-legacy.neurorobotics.ebrains.eu* by executing this command:
 
 .. code-block:: python
 
-    vc = VirtualCoach('http://148.187.149.212', oidc_username='<your-hbp-username>', oidc_password='<your-hbp-password>')
+    vc = VirtualCoach('https://nrp-legacy.neurorobotics.ebrains.eu', oidc_username='<your-hbp-username>', oidc_password='<your-hbp-password>')
 
 Notice that parameters *storage_username* and *storage_password* are replaced with *oidc_username* and *oidc_password*. The latter must be used when EBRAINS :term:`OIDC` authentication is required, the former are used otherwise.
 

hbp_nrp_virtual_coach/pynrp/config.py

@@ -56,6 +56,9 @@ class Config(dict):
         # initialize the parent dictionary, required by pylint
         dict.__init__(self)
 
+        # remove trailing slash in the environment url
+        url = environment[:-1] if environment.endswith('/') else environment
+
         # ensure the config.json file exists and is readable by our user
         logger.info('Loading configuration file config.json')
         path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.json')
@@ -90,7 +93,7 @@ class Config(dict):
         # convenience, prepend the proxy url to all proxy services, we cannot do this
         # for the simulation services because they are backend/experiment id specific
         for k, v in self['proxy-services'].items():
-            self['proxy-services'][k] = '{}/proxy/{}'.format(environment, v)
+            self['proxy-services'][k] = '{}/proxy/{}'.format(url, v)
 
     def __validate(self, key, values):
         """

hbp_nrp_virtual_coach/pynrp/simulation.py

@@ -885,7 +885,7 @@ class Simulation(object):
 
         :param file_name: The name of csv file for which to retrieve the content.
         """
-        return self.__vc.get_csv_last_run_file(self.__experiment_id, file_name)
+        return self.__vc.get_last_run_file(self.__experiment_id, 'csv', file_name)
 
     def __send_recorder_cmd(self, cmd):
         """

hbp_nrp_virtual_coach/pynrp/tests/test_config.py

@@ -65,6 +65,9 @@ class TestConfig(unittest.TestCase):
         for k, v in config['proxy-services'].items():
             self.assertEqual(v, '%s/proxy/%s' % ('localhost', self._conf['proxy-services'][k]))
 
+    def test_trailing_slash(self):
+        self.assertEqual(Config('localhost:8080'), Config('localhost:8080/'))
+
     @patch('pynrp.config.json.load')
     def test_validation_missing_key(self, mock_load):
         # missing section

hbp_nrp_virtual_coach/pynrp/tests/test_simulation.py

@@ -599,10 +599,10 @@ class TestSimulation(unittest.TestCase):
 
     def test_get_csv_last_run_file(self):
         self._sim._Simulation__vc = Mock()
-        self._sim._Simulation__vc.get_csv_last_run_file = Mock()
+        self._sim._Simulation__vc.get_last_run_file = Mock()
         self._sim._Simulation__experiment_id = 'experiment_id'
         self._sim.get_csv_last_run_file('file_name')
-        self._sim._Simulation__vc.get_csv_last_run_file.assert_called_once_with('experiment_id', 'file_name')
+        self._sim._Simulation__vc.get_last_run_file.assert_called_once_with('experiment_id', 'csv', 'file_name')
 
     def test_recording(self):
         self._sim._Simulation__config = {'simulation-services': {'recorder':'recorder'}}
@@ -623,4 +623,4 @@ class TestSimulation(unittest.TestCase):
         self._sim._Simulation__http_client.post.assert_called_with(u'url/recorder/reset', body='reset')
 
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()

hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py

@@ -29,6 +29,7 @@ from future import standard_library
 standard_library.install_aliases()
 from builtins import object
 from pynrp.virtual_coach import VirtualCoach
+import pynrp.version
 
 from unittest.mock import Mock, patch, MagicMock
 import unittest
@@ -149,6 +150,7 @@ class TestVirtualCoach(unittest.TestCase):
                                                       "sdf":"example_model_2.sdf",
                                                       "configPath":"example_model_2/model.config"}}
 
+
     def test_init_asserts_no_password(self):
         # invalid environment
         self.assertRaises(AssertionError, VirtualCoach, environment=True)
@@ -166,6 +168,14 @@ class TestVirtualCoach(unittest.TestCase):
     def test_no_login_credentials(self):
         self.assertRaises(Exception, VirtualCoach)
 
+    def test_version(self):
+        self.assertRegex(pynrp.version._get_version(), "^\d+.\d+.\d+")
+        self.assertRegex(pynrp.version.VERSION, "^\d+.\d+.\d+")
+
+    @patch('os.getenv', return_value="/non-existing-path")
+    def test_version_exception(self, mock_getenv):
+        self.assertRaises(RuntimeError, pynrp.version._get_version)
+
 
     @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_storage_token')
     @patch('getpass.getpass', return_value='password')

hbp_nrp_virtual_coach/pynrp/version.py

-'''version string - generated by setVersion.sh'''
-VERSION = '3.2.1'
+'''version string - automatically calculated from the SCM (git)'''
+import subprocess
+import os
+from subprocess import CalledProcessError
+
+def _get_version():
+    try:
+        version = subprocess.run(['bash',
+                            f'{os.getenv("HBP")}/user-scripts/nrp_get_scm_version.sh',
+                            'get_scm_version'],
+                            stdout=subprocess.PIPE, check=True).stdout.decode('utf-8')
+    except CalledProcessError as e:
+        raise RuntimeError("The SCM version calculation script failed.\
+            Expected path: $HBP/user-scripts/nrp_get_scm_version.sh,\
+            check its existance.") from e
+    return version
+
+VERSION = _get_version()

hbp_nrp_virtual_coach/requirements.txt

@@ -3,7 +3,7 @@ future
 texttable==0.8.7
 progressbar2==3.52.1
 httplib2==0.12.1
-pandas==1.0.5
+pandas==1.4.4
 jupyter==1.0.0
 requests
 ipykernel==5.3.2

hbp_nrp_virtual_coach/setup.py

@@ -2,47 +2,15 @@
 
 # pylint: disable=F0401,E0611,W0622,E0012,W0142,W0402
 
-from builtins import str
-try:
-    from setuptools import setup
-except ImportError:
-    from distutils.core import setup
-
-from optparse import Option  # pylint:disable=deprecated-module
-import pip
+from setuptools import setup
 import pathlib
 import pynrp
 
 README = (pathlib.Path(__file__).parent / "README.txt").read_text()
 
-options = Option('--workaround')
-options.skip_requirements_regex = None
 reqs_file = './requirements.txt'
-
-pip_version_major = int(pip.__version__.split('.')[0])
-# Hack for old pip versions
-if pip_version_major == 1:
-    # Versions 1.x rely on pip.req.parse_requirements
-    # but don't require a "session" parameter
-    from pip.req import parse_requirements # pylint:disable=no-name-in-module, import-error
-    install_reqs = parse_requirements(reqs_file, options=options)
-    reqs = [str(ir.req) for ir in install_reqs]
-elif 10 > pip_version_major > 1:
-    # Versions greater than 1.x but smaller than 10.x rely on pip.req.parse_requirements
-    # and requires a "session" parameter
-    from pip.req import parse_requirements # pylint:disable=no-name-in-module, import-error
-    from pip.download import PipSession  # pylint:disable=no-name-in-module, import-error
-    options.isolated_mode = False
-    install_reqs = parse_requirements(  # pylint:disable=unexpected-keyword-arg
-        reqs_file,
-        session=PipSession,
-        options=options
-    )
-    reqs = [str(ir.req) for ir in install_reqs]
-elif pip_version_major >= 10:
-    # Versions greater or equal to 10.x don't rely on pip.req.parse_requirements
-    install_reqs = list(val.strip() for val in open(reqs_file))
-    reqs = install_reqs
+install_reqs = list(val.strip() for val in open(reqs_file))
+reqs = install_reqs
 
 config = {
     'description': 'Python interface to the Neurorobotics Platform (NRP)',
@@ -57,7 +25,10 @@ config = {
     'package_data': {
         'pynrp': ['config.json']
     },
-    'classifiers': ['Programming Language :: Python :: 3.8'],
+    'classifiers': [
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10'],
     'scripts': [],
     'name': 'pynrp',
     'include_package_data': True,