From 8770b0d7f22c695c498148318e3f773ac5820d13 Mon Sep 17 00:00:00 2001
From: Axel von Arnim <axel.vonarnim@fortiss.org>
Date: Fri, 26 Oct 2018 14:01:27 +0000
Subject: [PATCH] Merged in NRRPLT-6586 (pull request #2)
NRRPLT-6586
* [NRRPLT-6858] Add pipeline
* [NRRPLT-6858] ExDBackend clone missing
* [NRRPLT-6858] wrong path in generated
* [NRRPLT-6858] remove brainsimulation egg-links
* [NRRPLT-6858] missing egg-links removals
* [NRRPLT-6858] Simplify PYTHONPATH
* [NRRPLT-6586] Add branch dependencies
* [NRRPLT-6586] Reinstate get_neuron_count
* [NRRPLT-6586] Make get_neuron_count static
* [NRRPLT-6586] Externalize topic branches to admin script
* [NRRPLT-6586] Externalize topic branches to admin script 2
* [NRRPLT-6586] link to ContinuousIntegration not needed anymore
* [NRRPLT-6586] Remove more eggs
* [NRRPLT-6858] Add pipeline
* [NRRPLT-6858] ExDBackend clone missing
* [NRRPLT-6858] wrong path in generated
* [NRRPLT-6858] remove brainsimulation egg-links
* [NRRPLT-6858] missing egg-links removals
* [NRRPLT-6858] Simplify PYTHONPATH
* [NRRPLT-6586] Add branch dependencies
* [NRRPLT-6586] Reinstate get_neuron_count
* [NRRPLT-6586] Make get_neuron_count static
* [NRRPLT-6586] Externalize topic branches to admin script
* [NRRPLT-6586] Externalize topic branches to admin script 2
* [NRRPLT-6586] link to ContinuousIntegration not needed anymore
* [NRRPLT-6586] Remove more eggs
* [NRRPLT-6586] Exclude music packages from the build
* [NRRPLT-6586] Exclude music packages from the build 2
* [NRRPLT-6586] Exclude music packages from the build 3
* [NRRPLT-6586] Exclude music packages from the build 4
Approved-by: Kepa Cantero <cantero@fortiss.org>
---
Makefile | 8 ++-
bitbucket-pipelines.yml | 52 +++++++++++++++++++
.../bibi/bibi_music_config.py | 26 ++++++++--
3 files changed, 81 insertions(+), 5 deletions(-)
create mode 100644 bitbucket-pipelines.yml
diff --git a/Makefile b/Makefile
index c66f025..f2d4b3e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,15 @@
#modules that have tests
-TEST_MODULES=hbp_nrp_music_xml/hbp_nrp_music_xml hbp_nrp_music_interface/hbp_nrp_music_interface hbp_nrp_distributed_nest/hbp_nrp_distributed_nest
+#TEST_MODULES=hbp_nrp_music_xml/hbp_nrp_music_xml hbp_nrp_music_interface/hbp_nrp_music_interface hbp_nrp_distributed_nest/hbp_nrp_distributed_nest
+# HOTFIX: as import music fails in tests (NRRPLT-6949), remove music packages from the tests
+TEST_MODULES=hbp_nrp_distributed_nest/hbp_nrp_distributed_nest
#modules that are installable (ie: ones w/ setup.py)
INSTALL_MODULES=hbp_nrp_music_xml hbp_nrp_music_interface hbp_nrp_distributed_nest
#packages to cover
-COVER_PACKAGES=hbp_nrp_music_xml hbp_nrp_music_interface hbp_nrp_distributed_nest
+#COVER_PACKAGES=hbp_nrp_music_xml hbp_nrp_music_interface hbp_nrp_distributed_nest
+# HOTFIX: as import music fails in tests (NRRPLT-6949), remove music packages from the tests
+COVER_PACKAGES=hbp_nrp_distributed_nest
#documentation to build
#DOC_MODULES=hbp_nrp_music_xml/doc hbp_nrp_music_interface/doc hbp_nrp_distributed_nest/doc
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
new file mode 100644
index 0000000..1474b1a
--- /dev/null
+++ b/bitbucket-pipelines.yml
@@ -0,0 +1,52 @@
+# This is a sample build configuration for Docker.
+# Check our guides at https://confluence.atlassian.com/x/O1toN for more examples.
+# Only use spaces to indent your .yml configuration.
+# -----
+# You can specify a custom docker image from Docker Hub as your build environment.
+image: hbpneurorobotics/nrp:dev
+
+pipelines:
+ default:
+ - step:
+ name: Build
+ script:
+ # Branch dependencies (ex-gerrit topic)
+ - git archive --remote=ssh://git@bitbucket.org/hbpneurorobotics/admin-scripts.git refs/heads/master nrp_branch_topic_checkout | tar xf -
+ - . ./nrp_branch_topic_checkout admin-scripts user-scripts ExDBackend ExperimentControl CLE Experiments Models
+ - cd $BITBUCKET_CLONE_DIR
+
+ # Configure build has to be placed before make devinstall
+ - export VIRTUAL_ENV_PATH=$VIRTUAL_ENV
+ - export NRP_INSTALL_MODE=dev
+ - export PYTHONPATH=hbp_nrp_music_xml:hbp_nrp_music_interface:$VIRTUAL_ENV_PATH/lib/python2.7/site-packages:$PYTHONPATH
+
+ # Concatenate all build requirements, ensure newline in between
+ - (echo; cat $HBP/ExperimentControl/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
+ - (echo; cat $HBP/CLE/hbp_nrp_cle/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
+ - (echo; cat $HBP/ExDBackend/hbp_nrp_commons/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
+
+ # Checkout config.ini.sample from user-scripts
+ - cp $HBP/user-scripts/config_files/CLE/config.ini.sample $HBP/CLE/hbp_nrp_cle/hbp_nrp_cle/config.ini
+
+ # Copy bbp-client from user-scripts (before make devinstall)
+ - cp -af $HBP/user-scripts/config_files/platform_venv/* $VIRTUAL_ENV_PATH/lib/python2.7/site-packages/
+
+ # Generate schemas
+ # Egg-links have to be removed because make devinstall set them up wrongly
+ - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-music-interface.egg-link hbp-nrp-music-xml.egg-link hbp-nrp-distributed-nest.egg-link && popd
+ - make devinstall # Otherwise it can't find pyxbgen
+ - export pyxb_version=`grep "pyxb" $HBP/ExDBackend/hbp_nrp_commons/requirements.txt`
+ - . $VIRTUAL_ENV_PATH/bin/activate && pip install ${pyxb_version} && pyxbgen -u $HBP/Experiments/bibi_configuration.xsd -m bibi_api_gen && pyxbgen -u $HBP/Experiments/ExDConfFile.xsd -m exp_conf_api_gen && pyxbgen -u $HBP/Models/robot_model_configuration.xsd -m robot_conf_api_gen && pyxbgen -u $HBP/Models/environment_model_configuration.xsd -m environment_conf_api_gen
+ - mv bibi_api_gen.py exp_conf_api_gen.py _sc.py robot_conf_api_gen.py environment_conf_api_gen.py $HBP/ExDBackend/hbp_nrp_commons/hbp_nrp_commons/generated
+ - touch $HBP/ExDBackend/hbp_nrp_commons/hbp_nrp_commons/generated/__init__.py
+ - deactivate
+
+ # Run tests
+ # HOTFIX: Exclude music packages from pylint because of NRRPLT-6949
+ - export IGNORE_LINT='platform_venv|hbp_nrp_music_xml|hbp_nrp_music_interface|hbp_nrp_music_xml/hbp_nrp_music_xml/schema/generated|migrations|nest'
+ # Egg-links have to be removed because make devinstall set them up wrongly
+ - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-music-interface.egg-link hbp-nrp-music-xml.egg-link hbp-nrp-distributed-nest.egg-link && popd
+ - . $VIRTUAL_ENV_PATH/bin/activate && source /opt/ros/kinetic/setup.$CURR_SHELL && echo "PYTHONPATH $PYTHONPATH" && make verify_base || { if [ -f pylint.txt ]; then echo "----------"; echo "PYLINT.TXT"; echo "----------";cat pylint.txt; fi; if [ -f pep8.txt ]; then echo "----------"; echo "PEP8.TXT"; echo "----------";cat pep8.txt; fi; exit 1; }
+
+ # Coverage check
+ - $HBP/admin-scripts/nrp_cobertura_check coverage.xml
diff --git a/hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/bibi_music_config.py b/hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/bibi_music_config.py
index 60550da..be3a2de 100644
--- a/hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/bibi_music_config.py
+++ b/hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/bibi_music_config.py
@@ -26,7 +26,6 @@ Provdes a class to generate a MUSIC configuration from a BIBI configuration.
"""
from hbp_nrp_commons.generated import bibi_api_gen # pylint:disable=no-name-in-module
-from hbp_nrp_commons.bibi_functions import get_neuron_count
from hbp_nrp_music_xml.schema.generated import music_xml
from hbp_nrp_music_xml.config.music_config import Application, MusicConfigWriter, MusicConfigPort
@@ -75,11 +74,11 @@ class MUSICConfiguration(object):
# construct the bidirectional xml port/synapse definitions
for bibi_population in bibi.brainModel.populations:
name = bibi_population.population
- width = get_neuron_count(bibi_population)
+ width = MUSICConfiguration.get_neuron_count(bibi_population)
music_population = self.__bibi_to_music_population(bibi_population)
# FIXME: [NRRPLT-4722] Workaround for lack of dynamic MUSIC ports, allow the user
- # <fudge> the size of the population for devices (the neurons will be frozen
+ # <fudge> the size of the population for devices (the neurons will be frozen
# by default, which should not impact performance (too much))
faux_w = width * MUSICConfiguration.FAUX_DYNAMIC_FUDGE_FACTOR
self.__add_port('%s_to_brain' % name, faux_w, 'CLE', 'BRAIN', name, music_population)
@@ -88,6 +87,27 @@ class MUSICConfiguration(object):
# empty dict of applications, CLE and BRAIN definitions are required
self.__applications = {}
+ @staticmethod
+ def get_neuron_count(neurons):
+ """
+ Gets the amount of neurons connected
+
+ :param neurons: The neuron selector
+ :return: The amount of neurons as int
+ """
+ if isinstance(neurons, bibi_api_gen.Index):
+ return 1
+ elif isinstance(neurons, bibi_api_gen.Range):
+ if neurons.step is None:
+ return neurons.to - neurons.from_
+ return (neurons.to - neurons.from_) / neurons.step
+ elif isinstance(neurons, bibi_api_gen.List):
+ return len(neurons.element)
+ elif isinstance(neurons, bibi_api_gen.Population):
+ return neurons.count
+ raise Exception("Neuron Count: Don't know how to process neuron selector "
+ + str(type(neurons)))
+
def add_application(self, name, binary, args, processes):
"""
Add a sending or receiving MUSIC application definition.
--
GitLab