Merged in NRRPLT-7739-remove-music (pull request #15)

[NRRPLT-7739] removed music support.

* [NRRPLT-7739] removed music support. music distributed packages kept as tar files.

* [NRRPLT-7739] small fixes

Approved-by: Ugo Albanese <ugo.albanese@santannapisa.it>
Approved-by: Manos Angelidis <angelidis@fortiss.org>

Makefile

 #modules that have tests
-#TEST_MODULES=hbp_nrp_music_xml/hbp_nrp_music_xml hbp_nrp_music_interface/hbp_nrp_music_interface hbp_nrp_distributed_nest/hbp_nrp_distributed_nest
-# HOTFIX: as import music fails in tests (NRRPLT-6949), remove music packages from the tests
 TEST_MODULES=hbp_nrp_distributed_nest/hbp_nrp_distributed_nest
 
 #modules that are installable (ie: ones w/ setup.py)
-INSTALL_MODULES=hbp_nrp_music_xml hbp_nrp_music_interface hbp_nrp_distributed_nest
+INSTALL_MODULES=hbp_nrp_distributed_nest
 
 #packages to cover
-#COVER_PACKAGES=hbp_nrp_music_xml hbp_nrp_music_interface hbp_nrp_distributed_nest
-# HOTFIX: as import music fails in tests (NRRPLT-6949), remove music packages from the tests
 COVER_PACKAGES=hbp_nrp_distributed_nest
 
 #documentation to build
-#DOC_MODULES=hbp_nrp_music_xml/doc hbp_nrp_music_interface/doc hbp_nrp_distributed_nest/doc
+#DOC_MODULES=hbp_nrp_distributed_nest/doc
 
 PYTHON_PIP_VERSION?=pip==9.0.3
 

bitbucket-pipelines.yml

@@ -19,12 +19,12 @@ pipelines:
           # Configure build has to be placed before make devinstall
           - export VIRTUAL_ENV_PATH=$VIRTUAL_ENV
           - export NRP_INSTALL_MODE=dev
-          - export PYTHONPATH=hbp_nrp_music_xml:hbp_nrp_music_interface:$VIRTUAL_ENV_PATH/lib/python2.7/site-packages:$PYTHONPATH
+          - export PYTHONPATH=$VIRTUAL_ENV_PATH/lib/python2.7/site-packages:$PYTHONPATH
 
           # Concatenate all build requirements, ensure newline in between
-          - (echo; cat $HBP/ExperimentControl/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
-          - (echo; cat $HBP/CLE/hbp_nrp_cle/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
-          - (echo; cat $HBP/ExDBackend/hbp_nrp_commons/requirements.txt) >> hbp_nrp_music_interface/requirements.txt
+          - (echo; cat $HBP/ExperimentControl/hbp_nrp_excontrol/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
+          - (echo; cat $HBP/CLE/hbp_nrp_cle/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
+          - (echo; cat $HBP/ExDBackend/hbp_nrp_commons/requirements.txt) >> hbp_nrp_distributed_nest/requirements.txt
 
           # Checkout config.ini.sample from user-scripts
           - cp $HBP/user-scripts/config_files/CLE/config.ini.sample $HBP/CLE/hbp_nrp_cle/hbp_nrp_cle/config.ini
@@ -34,7 +34,7 @@ pipelines:
 
           # Generate schemas
           # Egg-links have to be removed because make devinstall set them up wrongly
-          - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-music-interface.egg-link hbp-nrp-music-xml.egg-link hbp-nrp-distributed-nest.egg-link && popd
+          - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-distributed-nest.egg-link && popd
           - make devinstall # Otherwise it can't find pyxbgen
           - export pyxb_version=`grep "pyxb" $HBP/ExDBackend/hbp_nrp_commons/requirements.txt`
           - . $VIRTUAL_ENV_PATH/bin/activate && pip install ${pyxb_version} && pyxbgen -u $HBP/Experiments/bibi_configuration.xsd -m bibi_api_gen && pyxbgen -u $HBP/Experiments/ExDConfFile.xsd -m exp_conf_api_gen && pyxbgen -u $HBP/Models/robot_model_configuration.xsd -m robot_conf_api_gen && pyxbgen -u $HBP/Models/environment_model_configuration.xsd -m environment_conf_api_gen
@@ -43,11 +43,10 @@ pipelines:
           - deactivate
 
           # Run tests
-          # HOTFIX: Exclude music packages from pylint because of NRRPLT-6949
-          - export IGNORE_LINT='platform_venv|hbp_nrp_music_xml|hbp_nrp_music_interface|hbp_nrp_music_xml/hbp_nrp_music_xml/schema/generated|migrations|nest'
+          - export IGNORE_LINT='platform_venv|migrations|nest'
           # Egg-links have to be removed because make devinstall set them up wrongly
-          - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-music-interface.egg-link hbp-nrp-music-xml.egg-link hbp-nrp-distributed-nest.egg-link && popd
-          - . $VIRTUAL_ENV_PATH/bin/activate && source /opt/ros/kinetic/setup.$CURR_SHELL && echo "PYTHONPATH $PYTHONPATH" && make verify_base || { if [ -f pylint.txt ]; then echo "----------"; echo "PYLINT.TXT"; echo "----------";cat pylint.txt; fi; if [ -f pep8.txt ]; then echo "----------"; echo "PEP8.TXT"; echo "----------";cat pep8.txt; fi; exit 1; }
+          - pushd $VIRTUAL_ENV_PATH/lib/python2.7/site-packages && rm -f hbp-nrp-distributed-nest.egg-link && popd
+          - . $VIRTUAL_ENV_PATH/bin/activate && source /opt/ros/melodic/setup.$CURR_SHELL && echo "PYTHONPATH $PYTHONPATH" && make verify_base || { if [ -f pylint.txt ]; then echo "----------"; echo "PYLINT.TXT"; echo "----------";cat pylint.txt; fi; if [ -f pep8.txt ]; then echo "----------"; echo "PEP8.TXT"; echo "----------";cat pep8.txt; fi; exit 1; }
 
           # Coverage check
           - $HBP/admin-scripts/nrp_cobertura_check coverage.xml

hbp_nrp_distributed_nest/README.txt

-This package provides interfaces for standalone distributed Nest simulation without
-any MUSIC dependencies for the CLE/ExDBackend.
+This package provides interfaces for standalone distributed Nest simulation for the CLE/ExDBackend.
\ No newline at end of file
\ No newline at end of file

hbp_nrp_music_interface/MANIFEST.in

-include requirements.txt
\ No newline at end of file

hbp_nrp_music_interface/README.txt

-This package provides interfaces between the generic MUSIC/PyNN hbp_nrp_music_xml
-package and the CLE/ExDBackend by implementing and overriding required interfaces.

hbp_nrp_music_interface/hbp_nrp_music_interface/__init__.py

-"""
-This package contains NRP specific implementations utilizing the MUSIC XML configuration
-and PyNN packages.
-"""
-
-from hbp_nrp_music_interface.version import VERSION as __version__  # pylint: disable=W0611
-
-__author__ = 'Martin Schulze'

hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/__init__.py

-"""
-This package contains classes to translate from a parsed BIBI specification to
-a MUSIC launch file and XML connectivity specification.
-"""

hbp_nrp_music_interface/hbp_nrp_music_interface/bibi/bibi_music_config.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-Provdes a class to generate a MUSIC configuration from a BIBI configuration.
-"""
-
-from hbp_nrp_commons.generated import bibi_api_gen # pylint:disable=no-name-in-module
-from hbp_nrp_music_xml.schema.generated import music_xml
-from hbp_nrp_music_xml.config.music_config import Application, MusicConfigWriter, MusicConfigPort
-
-import os
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-
-class MUSICConfiguration(object):
-    """
-    This class offers a high-level API for assembling MUSIC configuration that contains
-    ports between the Closed-Loop-Engine (CLE) and MUSIC-enabled neuro-simulators.
-
-    The MUSIC configuration is assembled by converting a BIBI specification into a
-    synapse/neuron specification MUSIC XML representation and also produces a formatted
-    MUSIC script that can be used to launch the distributed simulation components.
-    """
-
-    # FIXME: [NRRPLT-4722] fudge factor for faux dynamic ports
-    FAUX_DYNAMIC_FUDGE_FACTOR = 5
-
-    def __init__(self, bibi):
-        """
-        Convert the given BIBI configuration into a MUSIC XML configuration.
-
-        :param bibi: A parsed BIBI file to convert.
-        """
-        assert isinstance(bibi, bibi_api_gen.BIBIConfiguration)
-
-        #TODO: The BIBI schema currently does not require population definitions,
-        #      but they are required to properly generate a BIBI->MUSIC mapping.
-        #      This check should be removed after the BIBI is updated and consistent
-        #      for Python and h5 brains.
-        if bibi.brainModel.populations is None or len(bibi.brainModel.populations) == 0:
-            raise Exception("This BIBI file cannot be used for multi-process simulations "
-                            "as it does not specify neuron populations in the brainModel "
-                            "section. Please contact neurorobotics@humanbrainproject.eu if "
-                            "you require assistance to update this file or restrict this "
-                            "experiment to single process mode.")
-
-        self.__conf_xml = music_xml.root()
-        self.__conf_script = None
-
-        # construct the bidirectional xml port/synapse definitions
-        for bibi_population in bibi.brainModel.populations:
-            name = bibi_population.population
-            width = MUSICConfiguration.get_neuron_count(bibi_population)
-            music_population = self.__bibi_to_music_population(bibi_population)
-
-            # FIXME: [NRRPLT-4722] Workaround for lack of dynamic MUSIC ports, allow the user
-             #        <fudge> the size of the population for devices (the neurons will be frozen
-            #        by default, which should not impact performance (too much))
-            faux_w = width * MUSICConfiguration.FAUX_DYNAMIC_FUDGE_FACTOR
-            self.__add_port('%s_to_brain' % name, faux_w, 'CLE', 'BRAIN', name, music_population)
-            self.__add_port('%s_to_cle' % name, width, 'BRAIN', 'CLE', name, music_population)
-
-        # empty dict of applications, CLE and BRAIN definitions are required
-        self.__applications = {}
-
-    @staticmethod
-    def get_neuron_count(neurons):
-        """
-        Gets the amount of neurons connected
-
-        :param neurons: The neuron selector
-        :return: The amount of neurons as int
-        """
-        if isinstance(neurons, bibi_api_gen.Index):
-            return 1
-        elif isinstance(neurons, bibi_api_gen.Range):
-            if neurons.step is None:
-                return neurons.to - neurons.from_
-            return (neurons.to - neurons.from_) / neurons.step
-        elif isinstance(neurons, bibi_api_gen.List):
-            return len(neurons.element)
-        elif isinstance(neurons, bibi_api_gen.Population):
-            return neurons.count
-        raise Exception("Neuron Count: Don't know how to process neuron selector "
-                        + str(type(neurons)))
-
-    def add_application(self, name, binary, args, processes):
-        """
-        Add a sending or receiving MUSIC application definition.
-
-        :param name: The application name, currently only CLE and BRAIN are supported.
-        :param binary: The binary to execute for this application.
-        :param args: A list of arguments to python.
-        :param processes: The number of processes to allocate for this application.
-        """
-        if name not in ['CLE', 'BRAIN']:
-            raise Exception("Invalid application name {name} specified, must be "
-                            "either CLE or BRAIN.".format(name=name))
-
-        if name in self.__applications:
-            raise Exception("Duplicate application definition for {name}, this does not "
-                            "appear to be a valid request.".format(name=name))
-
-        self.__applications[name] = Application(name, binary, args, int(processes))
-
-    def save(self, path):
-        """
-        Save the MUSIC XML and application configuration files to the specified directory.
-
-        :patam path: The path of the directory to save to.
-        """
-
-        if 'CLE' not in self.__applications or 'BRAIN' not in self.__applications:
-            raise Exception('Missing CLE or BRAIN application definitions, cannote create MUSIC'
-                            'configuration script!')
-
-        # write the xml connectivity specification to proxy.xml
-        proxy_file = os.path.join(path, 'proxy.xml')
-        with open(proxy_file, 'w') as f:
-            f.write(self.__conf_xml.toxml())
-
-        # write the actual music launching script to cle.music
-        music_file = os.path.join(path, 'cle.music')
-        with open(music_file, 'w') as f:
-            ports = [MusicConfigPort(p) for p in self.__conf_xml.port]
-            applications = [self.__applications['CLE'], self.__applications['BRAIN']]
-            self.__conf_script = MusicConfigWriter(None, applications, ports)
-            self.__conf_script.write(f)
-
-    @staticmethod # required to avoid "no self use" pylint error
-    def __bibi_to_music_population(bibi_population):
-        """
-        This function converts BIBI NeuronSelector object (population specification) into
-        a MUSIC-XML SynapticSelector objects. Currently only Range types are supported in
-        population definitions, but this function supports all potential types.
-
-        :param bibi_population: A BIBI MultiNeuronSelector object
-        """
-
-        def make_list_selector(iterable):
-            """
-            Helper function to create a list of neurons from multiple BIBI selector types.
-
-            :param iterable: A list of neuron ids to add to this list selector.
-            """
-            neuron_list = music_xml.ListSelector()
-            for element in iterable:
-                neuron_list.append(element)
-            return neuron_list
-
-        if isinstance(bibi_population, bibi_api_gen.Range):
-            neuron_slice = music_xml.SliceSelector()
-            neuron_slice.start = bibi_population.from_
-            neuron_slice.stop = bibi_population.to
-            step = getattr(bibi_population, 'step', 1)
-            if not step:
-                step = 1
-            neuron_slice.step = step
-            return neuron_slice
-
-        elif isinstance(bibi_population, bibi_api_gen.List):
-            return make_list_selector(bibi_population.element)
-
-        elif isinstance(bibi_population, bibi_api_gen.Population):
-            return make_list_selector(list(range(0, bibi_population.count)))
-
-        else:
-            raise Exception("The deduction from type {} (for population declaration "
-                            "'{}') to the MUSIC equivalent population declaration "
-                            "is not implemented".format(type(bibi_population),
-                                                        bibi_population.population))
-
-    def __add_port(self, name, width, sender, receiver, target, selector):
-        """
-        Generates a MUSIC XML spike event port for the named population with given width between
-        the specified source and target.
-
-        :param name: Base name of the MUSIC port (NOT the population name).
-        :param width: The MUSIC port width (number of neurons to
-        :param sender: The name of the producing MUSIC application (e.g. CLE, BRAIN)
-        :param receiver: The name of the receiving MUSIC application (e.g. CLE, BRAIN)
-        :param target: The neuron population name to connect to/from.
-        :param selector: The MUSIC XML population selector (e.g. slice/etc.)
-        """
-
-        if sender not in ['CLE', 'BRAIN'] or receiver not in ['CLE', 'BRAIN']:
-            raise Exception("Invalid sender {sender} or receiver {receiver} specified, must be "
-                            "either CLE or BRAIN.".format(sender=sender, receiver=receiver))
-
-        port = music_xml.Port()
-        port.type = music_xml.PortType.Event
-        port.name = name
-        port.width = width
-        port.sender = music_xml.Peer(name=sender)
-        port.receiver.append(music_xml.Peer(name=receiver))
-
-        synapse = music_xml.SynapticConnection()
-        synapse.type = music_xml.ConnectionType.one_to_one
-        synapse.target = target
-        synapse.selector = selector
-        port.receiver[0].synapse = [synapse]
-
-        logger.debug("Created MUSIC port for spike event proxy: "
-                     "{sender_name}.{port_name} ---/port_width:{port_width}/---> "
-                     "{receiver_name}.{port_name}"
-                     .format(sender_name=port.sender.name, port_width=port.width,
-                             receiver_name=port.receiver[0].name, port_name=port.name))
-
-        self.__conf_xml.port.append(port)
-
-    def __str__(self):
-        """
-        Returns the formatted MUSIC configuration script as a string.
-        """
-        return str(self.__conf_script)
-
-    def xml(self):
-        """
-        Returns the XML string representation of the underlying generated configuration.
-        """
-        return str(self.__conf_xml.toxml())

hbp_nrp_music_interface/hbp_nrp_music_interface/cle/MUSICBrainLoader.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""`
-Load a brain network's MUSIC proxies
-"""
-import pyNN.nest as sim
-import hbp_nrp_cle.tf_framework.config as tf_config
-
-from hbp_nrp_music_xml.pynn.factory import PyNNProxyFactory
-from hbp_nrp_music_xml.pynn.connector_factory import PyNNConnectorFactory
-from hbp_nrp_music_xml.pynn.xml_factory import XmlFactory
-
-import imp
-
-__brainIndex = 0
-
-
-# pylint: disable=global-statement
-def load_proxies_from_xml(xml_file, **populations):
-    """
-    Load neuron population proxy devices from the specified music_xml file and create a
-    mapping between population names and their proxy devices. This will allow the
-    communications adapter or any other components to lookup the appropriate input or
-    output proxy for a given named population.
-
-    :param xml_file: Path to the music_xml ile containing proxy population definitions.
-    :param populations: A named list of population proxies to create
-    """
-
-    # ensure a Python brain module has not been loaded or set
-    if tf_config.brain_root is not None:
-        raise Exception("A brain module has been loaded in this process, no brain should be"
-                        "present when loading MUSIC proxies. Invalid configuration, aborting.")
-
-    # attempt to read the XML file, abort if doesn't exist or cannot be read
-    try:
-        with open(xml_file, 'r') as f:
-            xml = f.read()
-    except IOError:
-        raise Exception("Cannot create MUSIC proxy devices for simulation, unable to access "
-                        "XML specification file {}. Aborting!".format(xml_file))
-
-    # simulator specific configuration (not through PyNN, make sure simulator is supported)
-    if sim.simulator.name != "NEST":
-        raise Exception("Unable to load MUSIC proxy devices for simulator {}, currently only "
-                        "NEST is supported. Aborting!".format(sim.simulator.name))
-
-    # build proxy factories and store a dictionary of built proxies
-    proxy_model_factory = PyNNProxyFactory(sim)
-    connector_factory = PyNNConnectorFactory(sim)
-    xml_factory = XmlFactory("CLE", connector_factory, proxy_model_factory, {})
-    proxy_dict = xml_factory.create_proxies(xml)
-
-    # create a surrogate python brain module in place of loading the actual brain module
-    global __brainIndex
-    tf_config.brain_root = imp.new_module('__brain_model{}'.format(__brainIndex))
-    __brainIndex += 1
-
-    # the CLE will only be able to access named proxy populations that were specified in the BIBI
-    # create a population -> proxy dictionary in the tf config so that the communications adapter
-    # can appropriately locate and use named input or output proxies as need
-    # store a proxy population in the root brain module for each population so that the existing
-    # tf framework can create population views without modification - understanding that the
-    # communications adapter will replace the base proxy with the input/output population as needed
-    try:
-        tf_config.music_proxies = {}
-
-        for p in populations:
-
-            # create population views to mimic how normal neuron populations are created
-            proxy_to_brain = proxy_dict['{}_to_brain'.format(p)]
-            proxy_to_brain_view = sim.PopulationView(proxy_to_brain,
-                                                     slice(0, len(proxy_to_brain), None),
-                                                     label=p)
-            proxy_to_cle = proxy_dict['{}_to_cle'.format(p)]
-            proxy_to_cle_view = sim.PopulationView(proxy_to_cle,
-                                                   slice(0, len(proxy_to_cle), None),
-                                                   label=p)
-
-            # default named proxy population, a named population for p must be in the brain_root
-            # dictionary for transfer functions to create slices, but the correct population for
-            # a source or sink will be properly selected in the communication adapter, so this
-            # specification is arbitrary but definitely required
-            tf_config.brain_root.__dict__[p] = proxy_to_brain_view
-
-            # store the named proxies by direction
-            tf_config.music_proxies[p] = {'source': proxy_to_brain_view, 'sink': proxy_to_cle_view}
-
-    except KeyError:
-        raise Exception("Could not find proxy-specification of population {} in MUSIC "
-                        "XML specification file: {}".format(p, xml_file))

hbp_nrp_music_interface/hbp_nrp_music_interface/cle/MUSICPyNNCommunicationAdapter.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-Extensions of the base CLE PyNNCommunicationAdapter to use MUSIC specific proxies
-instead of direct population access. Maxmimum code reuse and minimal duplication
-where possible.
-"""
-from hbp_nrp_cle.brainsim.pynn_nest.PyNNNestCommunicationAdapter import PyNNNestCommunicationAdapter
-import pyNN.nest as sim
-import hbp_nrp_cle.tf_framework.config as tf_config
-
-from hbp_nrp_music_interface.bibi.bibi_music_config import MUSICConfiguration
-from hbp_nrp_distributed_nest.launch.NestBrainProcess import NestBrainProcess
-
-from mpi4py import MPI
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class MUSICPyNNCommunicationAdapter(PyNNNestCommunicationAdapter):
-    """
-    Represents a MUSIC/PyNN proxy communication adapter for the neuronal simulation
-    """
-
-    def initialize(self):
-        """
-        Marks the MUSICPyNN adapter as initialized.
-        """
-        logger.info("MUSICPyNN communication adapter initialized")
-        super(MUSICPyNNCommunicationAdapter, self).initialize()
-
-    def register_spike_source(self, populations, spike_generator_type, **params):
-        """
-        Intercepts default PyNNNestCommunicationAdapter request and replaces specified
-        population with a new set of proxy neurons for each individual TF. This is required
-        to ensure weighting between the TF and target node is maintained properly as multiple
-        TFs can target a single neuron and we must weight their inputs separately.
-
-        This function is over-complicated due to backwards comaptibility for the single process
-        CLE, it can be simplified in the future if we merge these repositories and especially
-        when MUSIC supports dynamic ports.
-
-        :param populations: A reference to the populations to which the spike generator
-                            should be connected
-        :param spike_generator_type: A spike generator type (see documentation
-                                     or a list of allowed values)
-        :param params: A dictionary of configuration parameters
-        :return: A communication object or a group of objects
-        """
-        # get the next set of free parrot neurons with real connectivity to proxy_out neuron
-        proxies = self.__get_population_proxy(populations, 'source')
-
-        # connect the device generator to the new proxy neurons, this will create synapses with
-        # given parameters that we will need to duplicate from proxy->real neuron (parrot neurons
-        # ignore their input synapses, but output synapses are respected)
-        device = super(MUSICPyNNCommunicationAdapter, self). \
-                       register_spike_source(proxies, spike_generator_type, **params)
-
-        # notify the remote brain processes that they need to setup the other side of this proxy
-        self.__notify_brain_processes(populations, proxies)
-
-        # return the new generator device
-        return device
-
-    def register_spike_sink(self, populations, spike_detector_type, **params):
-        """
-        Requests a communication object with the given spike detector type
-        for the given set of neurons
-
-        :param populations: A reference to the populations which should be connected
-                            to the spike detector
-        :param spike_detector_type: A spike detector type (see documentation
-                                    for a list of allowed values)
-        :param params: A dictionary of configuration parameters
-        :return: A Communication object or a group of objects
-        """
-        populations = self.__get_population_proxy(populations, 'sink')
-        return super(MUSICPyNNCommunicationAdapter, self). \
-                  register_spike_sink(populations, spike_detector_type, **params)
-
-    @staticmethod
-    def __notify_brain_processes(populations, proxies):
-        """
-        Notify remote MPI Brain Processes that they must complete this transfer function
-        connection by duplicating the parameters for this device connection with the output
-        proxy neurons. This is the only way to ensure a TF is actually connected with the
-        right parameters.
-
-        :param populations The target population to connect to on the other side.
-        :param proxies The proxy/parrot neurons that the device is actually connected to.
-        """
-
-        # nest is the only supported simulator in the hbp_nrp_music packages, but assert here
-        # in case this changes, import here to avoid starting nest by accident earlier
-        assert sim.simulator.name == "NEST", "NEST is currently required to reconfigure MUSIC ports"
-        import nest
-
-        # synapse details to pass to other MPI clients, the synapse parameters should all be
-        # the same, but in the future a device may have variable/randomly parameterized
-        # synapses so just support them now
-        synapse_params = []
-
-        # for each proxy/reference pair, extract input connectivity from the device, the real
-        # reference output targets, and finally duplicate the input connectivity to the targets
-        for p in map(int, proxies.all_cells):
-
-            # the reference connection from the parrot neuron to proxy out, this contains music
-            # channel information
-            ref_c = nest.GetStatus(nest.GetConnections(source=[p]))[0]
-            channel = ref_c['receptor']
-
-            # the connection from device to parrot neuron, our desired synapse
-            dev_c = nest.GetStatus(nest.GetConnections(target=[p]))[0]
-            model = str(dev_c['synapse_model'])
-
-            # remove specific connection information, only leave synapse params since
-            # otherwise nest will complain about unused values
-            for k in ['receptor', 'source', 'target', 'synapse_label', 'synapse_model', 'sizeof']:
-                if k in dev_c:
-                    dev_c.pop(k)
-
-            # override the original dummy weighted parrot->proxy synapse, we can't create a
-            # new synapse because MUSIC will complain about duplicate use of music_channels
-            nest.SetStatus(nest.GetConnections(source=[p]), dev_c)
-
-            # store the connection parameters for remote clients
-            dev_c['model'] = model
-            dev_c['music_channel'] = channel
-            synapse_params.append(dev_c)
-
-        # population information to send to the remote MPI nodes, we can't pickle Populations
-        # directly and those references wouldn't be valid on the remote nodes anyway
-        label = populations.parent.label if populations.parent else populations.label
-
-        # [NRRPLT-4722] workaround, if there is no population mask, then we need to use the size
-        # of the proxies to target the real size of the neuron population instead of the inflated
-        # proxy size
-        mask = populations.mask if populations.mask else slice(0, len(proxies), 1)
-
-        # propagate the synapse creation parameters to all remote notes, they will create the
-        # other side of the connections for this type
-        for rank in xrange(MPI.COMM_WORLD.Get_size()):
-            if rank == MPI.COMM_WORLD.Get_rank():
-                continue
-
-            MPI.COMM_WORLD.send({'command': 'ConnectTF', 'label': label, 'mask': mask,
-                                 'synapses': synapse_params},
-                                dest=rank, tag=NestBrainProcess.MPI_MSG_TAG)
-
-    def __get_population_proxy(self, population, proxy_type):
-        """
-        Retrieves the first <size> free proxy neurons of the specified population proxy type
-        for a given population.
-
-        :param population The population to find/create a proxy for.
-        :param proxy_type string The type of proxy device to retrieve.
-        :return: An equivalend population list or view containing proxies.
-        """
-        if proxy_type not in ['sink', 'source']:
-            raise TypeError("Cannot retrieve unknown population proxy type: {}" % proxy_type)
-
-        # lists are explicitly allowed by __register_device, support even if they are unused
-        if isinstance(population, list):
-            return [self.__get_population_proxy(p, proxy_type) for p in population]
-
-        assert isinstance(population, sim.PopulationView)
-        try:
-
-            # for sinks we must use the same few neurons that are allocated at MUSIC launch
-            # otherwise the frontend will not properly fill out the neuron monitor and things
-            if proxy_type == 'sink':
-                # top level population view with no subslicing (e.g. sensors, actors, etc.)
-                if isinstance(population.parent, sim.Population):
-                    # tf_config.music_proxies is set in
-                    # hbp_nrp_music_interface/hbp_nrp_music_interface/cle/MUSICBrainLoader.py
-                    # and not in hbp_nrp_cle.tf_framework.config
-                    # which confuses pylint
-                    # pylint: disable=no-member
-                    return tf_config.music_proxies[population.label][proxy_type]
-                    # pylint: enable=no-member
-
-                # otherwise, this is a view of a top level named population view
-                # pylint: disable=no-member
-                parent = tf_config.music_proxies[population.parent.label][proxy_type]
-                # pylint: enable=no-member
-                return sim.PopulationView(parent, population.mask, population.label)
-
-            # [NRRPLT-4722] workaround, simply get the first <size> proxy neurons that are
-            # frozen for a population, unfreeze and return them for connection, for full
-            # population requests only get the real population size
-            import nest
-            # top level population view with no subslicing (e.g. sensors, actors, etc.)
-            # pylint: disable=no-member
-            if isinstance(population.parent, sim.Population):
-                proxies = tf_config.music_proxies[population.label][proxy_type]
-                size = proxies.size / MUSICConfiguration.FAUX_DYNAMIC_FUDGE_FACTOR
-
-            # otherwise, this is a view of a top level named population view
-            else:
-                proxies = tf_config.music_proxies[population.parent.label][proxy_type]
-                size = population.size
-            # pylint: disable=no-member
-            # find all of the free/frozen proxy neurons
-            mask = []
-            for i, p in enumerate(map(int, proxies.all_cells)):
-
-                # if the neuron is frozen, unfreeze it and add to our list
-                if nest.GetStatus([p], 'frozen')[0]:
-                    nest.SetStatus([p], 'frozen', False)
-                    mask.append(i)
-
-                # stop looping when we have enough free neurons
-                if len(mask) == size:
-                    break
-
-            # make sure we have enough free proxy neurons
-            if len(mask) != size:
-                raise Exception("Not enough free proxy neurons to connect transfer functions!\n"
-                                "Please contact neurorobotics@humanbrainproject.eu with details of "
-                                "this experiment and associated BIBI file  if you require support.")
-
-            # return a view of the free proxy neurons for connection
-            return sim.PopulationView(proxies, mask, population.label)
-
-        except KeyError:
-            raise Exception("Unable to locate distriuted MUSIC neuron population proxies for {}.\n"
-                            "This likely means that a transfer function is utilizing a neuron "
-                            "population that is not specified at the top of the BIBI file or is"
-                            "attempting to directly access the underlying \"circuit\" population."
-                            "Please contact neurorobotics@humanbrainproject.eu with details of "
-                            "this experiment and associated BIBI file  if you require support."
-                            .format(population))

hbp_nrp_music_interface/hbp_nrp_music_interface/cle/MUSICPyNNControlAdapter.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-Extensions of the base CLE PyNNControlAdapter to use MUSIC specific proxies
-instead of direct population access. Maxmimum code reuse and minimal duplication
-where possible.
-"""
-from hbp_nrp_distributed_nest.cle.DistributedPyNNControlAdapter import DistributedPyNNControlAdapter
-from hbp_nrp_music_interface.cle import MUSICBrainLoader
-
-import music
-import logging
-import os
-
-logger = logging.getLogger(__name__)
-
-
-class MUSICPyNNControlAdapter(DistributedPyNNControlAdapter):
-    """
-    Represents a MUSIC/PyNN proxy controller object for the neuronal simulator
-    """
-
-    def load_brain(self, network_file, **populations):
-        """
-        Load MUSIC/PyNN brain proxy populations rather than loading and instantiating
-        the brain - overrides functionality for both python and h5 brains.
-
-        :param network_file: The path to the python file containing the network
-        :param populations: A named list of populations to create
-        """
-        self.__load_music_brain_proxies(**populations)
-
-        # load the brain source for the frontend to display, copied from parent class
-        logger.info("Saving brain source")
-        import hbp_nrp_cle.tf_framework.config as tf_config
-        with open(network_file) as source:
-            tf_config.brain_source = source.read()
-
-    @staticmethod
-    def __load_music_brain_proxies(**populations):
-        """
-        Load MUSIC proxy devices for the given brain network specification and create
-        mappings from populations to input/output proxy devices.
-
-        :param populations: A named list of populations to create
-        """
-
-        # initialize MUSIC and load proxies from the MUSIC proxy xml
-        logger.info("Loading MUSIC proxy brain devices.")
-        music.Setup()
-        music_path = os.environ.get('NRP_MUSIC_DIRECTORY')
-        proxy_file = os.path.join(music_path, 'proxy.xml')
-        MUSICBrainLoader.load_proxies_from_xml(proxy_file, **populations)

hbp_nrp_music_interface/hbp_nrp_music_interface/cle/__init__.py

-"""
-This package contains an extended implementation of the CLE PyNN Brain Control Adapter
-and Brain Loader that use MUSIC proxy interfaces rather than direct access.
-"""
-
-__author__ = 'Martin Schulze'

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/MUSICBrainProcess.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-A distributed MUSIC brain process that can be launched standalone on remote hosts.
-"""
-import hbp_nrp_cle.brainsim.pynn_nest # pylint: disable=unused-import
-
-from hbp_nrp_distributed_nest.launch.NestBrainProcess import NestBrainProcess
-
-import pyNN.nest as sim
-import hbp_nrp_cle.tf_framework.config as tf_config
-
-from hbp_nrp_music_xml.pynn.factory import PyNNProxyFactory
-from hbp_nrp_music_xml.pynn.connector_factory import PyNNConnectorFactory
-from hbp_nrp_music_xml.pynn.xml_factory import XmlFactory
-
-import argparse
-import os
-import music
-
-from mpi4py import MPI
-import traceback
-
-
-class MUSICBrainProcess(NestBrainProcess):
-    """
-    A distributed MUSIC brain process that can be launched standalone on remote hosts.
-    """
-
-    def __init__(self, bibi_file, rng_seed):
-        """
-        Load the distributed brain and construct MUSIC proxies for communication
-        to/from the CLE. Nest will automatically allocate the brain in a round-robin
-        fashion under the hood, we do not need to do anything explicitly.
-
-        :param bibi_file The absolute path to the BIBI file for this experiment.
-        """
-
-        # setup MUSIC before any brain loading attempt
-        self._music_setup = music.Setup()
-
-        # load the brain and set parameters
-        super(MUSICBrainProcess, self).__init__(bibi_file, rng_seed)
-
-        # load the MUSIC proxies for the spawned brain
-        music_path = os.environ.get('NRP_MUSIC_DIRECTORY')
-        proxy_file = os.path.join(music_path, 'proxy.xml')
-        with open(proxy_file, 'r') as f:
-            music_xml = f.read()
-
-        proxy_model_factory = PyNNProxyFactory(sim)
-        connector_factory = PyNNConnectorFactory(sim)
-        xml_factory = XmlFactory("BRAIN",
-                                 connector_factory,
-                                 proxy_model_factory,
-                                 tf_config.brain_root.__dict__)
-
-        self._proxies = xml_factory.create_proxies(music_xml)
-
-    def _connect_tf(self, params):
-        """
-        Reflect a transfer function connection made on the CLE side by connecting proxy neurons
-        to real brain neurons using the same parameters and connectivity as the CLE. This is the
-        only way to guarantee both sides share the same connectivity using static port allocation.
-
-        :param params The connectivity/synapse parameters passed by the CLE.
-        """
-
-        # connections only supported during simulation construction
-        if self._ready:
-            raise Exception("The distributed MUSIC-Nest implementation does not dynamic TFs!")
-
-        # get the population of neurons from our dictionary, we can guarantee this is a valid
-        # population that has been declared in the BIBI at this point as the CLE will validate
-        # before sending us the create message
-        brain_pop = sim.PopulationView(tf_config.brain_root.__dict__[params['label']],
-                                       selector=params['mask'])
-
-        # get the whole population of proxy neurons for this port
-        port_name = '%s_to_brain' % params['label']
-        proxies = self._proxies[port_name]
-
-        # this is Nest specific code, but so is the rest of the pipeline
-        assert sim.simulator.name == "NEST", "Currently only NEST is supported"
-        import nest
-
-        # iterate through synapses and connect the specific proxy neuron (via music channel) to real
-        # brain neuron with given parameters
-        for synapse, brain_neuron in zip(params['synapses'], map(int, brain_pop.all_cells)):
-
-            # get the proxy neuron at the channel index
-            proxy = proxies[synapse['music_channel']]
-            synapse.pop('music_channel')
-
-            # thaw the proxy neuron so we actually get data
-            nest.SetStatus([proxy], 'frozen', False)
-
-            # for a source, we only need to connect the input proxy to the real neuron
-            nest.Connect([proxy], [brain_neuron], syn_spec=synapse)
-
-    def _delete_tf(self, params):
-        """
-        Currently unsupported, unable to dynamically create or destroy MUSIC ports.
-        """
-
-        # ignore any commands during simulation construction
-        if not self._ready:
-            return
-
-        raise Exception("The distributed MUSIC-Nest implementation does not support TF deletion!")
-
-    def _load_brain(self, params):
-        """
-        Currently unsupported, unable to dynamically create or destroy MUSIC ports.
-        """
-
-        # ignore any commands during simulation construction
-        if not self._ready:
-            return
-
-        raise Exception("The distributed MUSIC-Nest implementation does not support brain changes!")
-
-
-if __name__ == '__main__': # pragma: no cover
-
-    try:
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--bibi-file', dest='bibi_file',
-                            help='the bibi file path to load', required=True)
-        parser.add_argument('--rng-seed', dest='rng_seed',
-                            help='the global experiment RNG seed', required=True)
-        args = parser.parse_args()
-
-        # construct brain and proxies (expand environment variables in paths)
-        brain = MUSICBrainProcess(os.path.expandvars(args.bibi_file), args.rng_seed)
-
-        # run the brain until terminated, this is a blocking call
-        brain.run()
-
-    except Exception: # pylint: disable=broad-except
-        # print the traceback which should go back to the remote logger
-        traceback.print_exc()
-
-        # for any failures, terminate all other brain processes and the CLE
-        MPI.COMM_WORLD.Abort(-1)

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/MUSICCLEProcess.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-This module contains the CLE process logic for the simulation assembly using MUSIC
-"""
-
-import logging
-import sys
-
-from hbp_nrp_cleserver.server.CLEGazeboSimulationAssembly import CLEGazeboSimulationAssembly
-from hbp_nrp_cleserver.server.ServerConfigurations import robot_gazebo_ros_adapters
-from hbp_nrp_music_interface.cle.MUSICPyNNCommunicationAdapter import MUSICPyNNCommunicationAdapter
-from hbp_nrp_music_interface.cle.MUSICPyNNControlAdapter import MUSICPyNNControlAdapter
-
-from hbp_nrp_distributed_nest.launch.DistributedCLEProcess import launch_cle
-
-# maintain this import order and both must be done before mpi4py
-import hbp_nrp_cle.brainsim.pynn_nest # pylint: disable=unused-import
-import pyNN.nest as nestsim
-import music
-
-logger = logging.getLogger(__name__)
-
-
-class MusicCLESimulationAssembly(CLEGazeboSimulationAssembly):
-    """
-    Defines the assembly of a simulation using MUSIC
-    """
-
-    def __init__(self, sim_id, exc, bibi_model, **par):
-        """
-        Creates a new simulation assembly to simulate an experiment using the CLE and Gazebo
-        :param sim_id: The simulation id
-        :param exc: The experiment configuration
-        :param bibi_model: The BIBI configuration
-        """
-        super(MusicCLESimulationAssembly, self).__init__(sim_id, exc, bibi_model, **par)
-
-    def _create_robot_adapters(self):
-        """
-        Creates the adapter components for the robot side
-
-        :return: A tuple of the communication and control adapter for the robot side
-        """
-        return robot_gazebo_ros_adapters()
-
-    def _create_brain_adapters(self):
-        """
-        Creates the adapter components for the neural simulator
-
-        :return: A tuple of the communication and control adapter for the neural simulator
-        """
-        logger.info('Using MUSIC configuration and adapters for CLE')
-
-        # initialize music and set the CLE to use MUSIC adapters
-        music.Setup()
-
-        braincomm = MUSICPyNNCommunicationAdapter()
-        braincontrol = MUSICPyNNControlAdapter(nestsim)
-        return braincomm, braincontrol
-
-if __name__ == '__main__':  # pragma: no cover
-
-    # guaranteed to only be launched in one process by MUSIC, launch the CLE with defined assembly
-    launch_cle(sys.argv[1:], MusicCLESimulationAssembly)

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/MUSICLauncher.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-Setup, build, and launch a distributed MUSIC instance that will spawn the CLE and
-requested brain processes.
-"""
-from hbp_nrp_distributed_nest.launch.NestLauncher import NestLauncher
-
-from hbp_nrp_music_interface.bibi import bibi_music_config
-from hbp_nrp_music_interface.launch.MUSICMPILauncher import MUSICMPILauncher
-from hbp_nrp_music_interface.launch.host.LocalLauncher import LocalLauncher
-from hbp_nrp_music_interface.launch.host.LuganoLauncher import LuganoLauncher
-
-import os
-
-
-# This class intentionally does not inherit SimulationServer (even though it is an implementation of
-# it) in order to avoid duplicate notificators
-class MUSICLauncher(NestLauncher):
-    """
-    Setup, build, and launch a distributed MUSIC instance that will spawn the CLE and
-    requested brain processes.
-    """
-
-    def __init__(self, sim_id, exc, bibi, **par):
-        """
-        Store all experiment configuration parameters so that they can be propagated
-        to the remote hosts.
-
-        :param exc: the experiment configuration
-        :param bibi: the BIBI configuration.
-        :param server_host Target Gazebo/brain process host (e.g. local or lugano)
-        :param reservation Reservation string for cluster backend (None is a valid option)
-        :param sim_id The id of the simulation/experiment to be launched.
-        :param timeout The default simulation timeout (time initially allocated).
-        """
-        super(MUSICLauncher, self).__init__(sim_id, exc, bibi, **par)
-
-    # we should call the except_hook when something goes wrong in the simulation, but currently
-    # we don't
-    # pylint: disable=unused-argument
-    def initialize(self, environment_file, except_hook):
-        """
-        Construct the MUSIC launch configuration that will spawn CLE + brain processes
-        on distributed hosts.
-        """
-
-        # extract the environment file path
-        nrp_models_path = os.environ.get('NRP_MODELS_DIRECTORY').rstrip('/')
-        self._env_file = environment_file.replace(nrp_models_path, '$NRP_MODELS_DIRECTORY')
-
-        # create a host specific launcher
-        if self._server_host == 'local':
-            self._launcher = LocalLauncher()
-        elif self._server_host == 'lugano':
-            self._launcher = LuganoLauncher(self._exc.bibiConf.processes + 1,
-                                            self._timeout,
-                                            self._reservation)
-        else:
-            raise Exception('Unknown server host {}, cannot configure and launch MUSIC!'
-                            .format(self._server_host))
-
-        # create launch scripts for the CLE and brain processes
-        cle_launcher, brain_launcher = self._launcher.create_launch_scripts()
-
-        # command line argument friendly versions of timeout and reservation arguments
-        # the receiving processes must understand how to convert these back
-        reservation_str = self._reservation if self._reservation else ''
-        timeout_str = str(self._timeout).replace(' ', '_')
-
-        # build a MUSIC configuration script with correct brain ports, launchers and arugments
-        # save it to the host launcher temp directory, this is the same for every host
-        music_conf = bibi_music_config.MUSICConfiguration(self._bibi)
-        music_conf.add_application('CLE',
-                                   cle_launcher,
-                                   ['--exdconf={}'.format(self._exd_file),
-                                    '--bibi={}'.format(self._bibi_file),
-                                    '--environment={}'.format(self._env_file),
-                                    '--experiment-path={}'.format(self._exp_path),
-                                    '--gzserver-host={}'.format(self._server_host),
-                                    '--reservation={}'.format(reservation_str),
-                                    '--sim-id={}'.format(self._sim_id),
-                                    '--timeout={}'.format(timeout_str),
-                                    '--rng-seed={}'.format(self._rng_seed)],
-                                   1)
-        music_conf.add_application('BRAIN',
-                                   brain_launcher,
-                                   ['--bibi-file={}'.format(self._bibi_file),
-                                    '--rng-seed={}'.format(self._rng_seed)],
-                                   self._exc.bibiConf.processes)
-        music_conf.save(self._launcher.local_tmpdir)
-
-        # construct the actual MPI launcher
-        self.mpilauncher = MUSICMPILauncher('music cle.music')
-
-        # build and deploy configuration
-        self._build()

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/MUSICMPILauncher.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-
-"""
-Helper class to build and execute a formatted mpirun command for music in the format:
-
-  mpirun -envlist <vars> -np <proc> -host <hostname/ip> -wdir <temporary work dir> <command> : ...
-
-where each of the hosts has a specific working directory with necessary config files already
-in place. Also passes environment variables required for NRP/CLE execution.
-"""
-from hbp_nrp_distributed_nest.launch.MPILauncher import MPILauncher
-
-
-class MUSICMPILauncher(MPILauncher):
-    """
-    Class constructs and executes the MUSIC mpi launch command.
-    """
-
-    def __init__(self, executable):
-        super(MUSICMPILauncher, self).__init__(executable)
-
-    def add_host(self, hostname, tmpdir, processes=1):
-        """
-        Add a target host to the mpi launch configuration with MUSIC working directory set.
-
-        :param hostname The remote host name or ip.
-        :param tmpdir A valid temporary directory on the remote host to launch in.
-        :param processes The number of processes for this host.
-        """
-        self._hosts.append('-np {p} -host {h} -wdir {t} -genv NRP_MUSIC_DIRECTORY {t}'
-                           .format(p=processes, h=hostname, t=tmpdir))

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/__init__.py

-"""
-This package contains classes related to launching a distributed
-MUSIC/PyNN/NEST simulation within the NRP.
-"""

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/host/LocalLauncher.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-localhost launch target configuration.
-"""
-
-from hbp_nrp_distributed_nest.launch.host.LocalLauncher import LocalLauncher as ILocalLauncher
-
-import os
-import stat
-import sys
-
-
-class LocalLauncher(ILocalLauncher):
-    """
-    This launch configuration targets the localhost for all processes and is suitable for local
-    installs or deployed installs where the newly spawned processes can run on the same host
-    as the REST backend.
-    """
-
-    def __init__(self):
-        """
-        Create a local launcher to target localhost and create a local temporary directory to
-        write MUSIC configuration files.
-        """
-        super(LocalLauncher, self).__init__()
-
-    def create_launch_scripts(self):
-        """
-        Create a set of launch scripts for the CLE and individual brain processes with specific
-        implementations required for each host. Write the launch scripts to the temporary directory
-        for this launcher.
-
-        Returns a tuple (path to CLE launch script, path to BrainProcess launc script).
-        """
-        # create .sh launcher scripts for the CLE and brain processes
-        cle_launcher = self._create_launch_script('music_cle_launcher.sh',
-                                                  'hbp_nrp_music_interface.launch.MUSICCLEProcess')
-        brain_launcher = self._create_launch_script('music_brain_launcher.sh',
-                                                    'hbp_nrp_music_interface.launch.' +
-                                                    'MUSICBrainProcess')
-        return (cle_launcher, brain_launcher)
-
-    def _create_launch_script(self, name, module):
-        """
-        Create an executable script in the working temporary folder to launch the specified
-        Python module. These will be used by the MUSIC runtime on each of the hosts since there
-        are some quirks in launching "python -m <module> with <args>" directly throug MUSIC.
-
-        :param name The name for the launch script to create.
-        :param module The python module to launch with this script.
-        :return The absolute path to this launch script.
-        """
-
-        # absolute path to script in tmpdir
-        path = os.path.join(self._local_tmpdir, name)
-
-        # use the absolute path of the Python interpreter for our current process, this current
-        # process will not be executed through uwsgi, so this will be the correct interpreter
-        with open(path, 'w') as f:
-            f.write('#!/bin/bash\n')
-            f.write('{python} -m {module}\n'.format(python=sys.executable, module=module))
-
-        os.chmod(path, stat.S_IRWXU)
-
-        # return a relative path to the script, it's guaranteed to be run in the tmpdir
-        return './%s' % name

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/host/LuganoLauncher.py

-# ---LICENSE-BEGIN - DO NOT CHANGE OR MOVE THIS HEADER
-# This file is part of the Neurorobotics Platform software
-# Copyright (C) 2014,2015,2016,2017 Human Brain Project
-# https://www.humanbrainproject.eu
-#
-# The Human Brain Project is a European Commission funded project
-# in the frame of the Horizon2020 FET Flagship plan.
-# http://ec.europa.eu/programmes/horizon2020/en/h2020-section/fet-flagships
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-# ---LICENSE-END
-"""
-Lugano vizcluster launch target configuration.
-"""
-
-from hbp_nrp_music_interface.launch.host.LocalLauncher import LocalLauncher
-
-from hbp_nrp_commons.cluster.LuganoVizCluster import LuganoVizCluster
-
-from hbp_nrp_cle.cle import config
-
-import netifaces
-import os
-import stat
-
-
-class LuganoLauncher(LuganoVizCluster, LocalLauncher):
-    """
-    This launch configuration targets the Lugano vizcluster and handles brain/CLE process
-    allocation and distribution. This is executed before the CLE is launched, so launching will
-    immediately terminate if there are not enough cluster resources.
-    """
-
-    def __init__(self, processes, timeout, reservation):
-        """
-        Immediately attempt to allocate cluster resources for the brain processes. If this fails
-        or causes Exceptions, it will be propagated up to the user appropriately.
-
-        :param processes The total number of processes (brain + CLE) to reserve.
-        :param timeout The simulation timeout required by the vizcluster launcher.
-        :param reservation Resource reservation string to access reserved nodes.
-        """
-
-        # ensure both constructors are called, raise Exception if allocation fails
-        try:
-            super(LuganoLauncher, self).__init__(processes, 0, timeout.tzinfo, reservation)
-            self._allocate_job(reuse_nodes=True) # multiple brains can run on the same node
-        finally:
-            LocalLauncher.__init__(self)
-
-        # override hostname with allocated node, remote tmp does not exist until written
-        self._hostname = self._node
-        self._host_tmpdir = None
-
-    def _create_launch_script(self, name, module):
-        """
-        Create an executable script in the working temporary folder to launch the specified
-        Python module. These will be used by the MUSIC runtime on each of the hosts since there
-        are some quirks in launching "python -m <module> with <args>" directly throug MUSIC.
-
-        Handle specific vizcluster configuration (modules and environment).
-
-        :param name The name for the launch script to create.
-        :param module The python module to launch with this script.
-        :return The absolute path to this launch script on the remote host.
-        """
-
-        # absolute path to script in local tmpdir
-        path = os.path.join(self._local_tmpdir, name)
-
-        # determine if we are running on a dev or staging environment
-        environment = os.environ.get('ENVIRONMENT')
-
-        # set the ROS master uri to use an actual IP instead of localhost
-        ifaddress = netifaces.ifaddresses(config.config.get('network', 'main-interface'))
-        local_ip = ifaddress[netifaces.AF_INET][0]['addr']
-        ros_master_uri = os.environ.get("ROS_MASTER_URI").replace('localhost', local_ip)
-
-        # create a launch script that configures the vizcluster environment properly
-        with open(path, 'w') as f:
-            f.write('#!/bin/bash\n')
-            f.write('source /opt/rh/python27/enable\n')
-
-            # set the terminal type to ensure we get the same behavior as from a backend VM
-            # this is also required in the Docker images
-            f.write('export TERM=linux\n')
-
-            # set the environment version and any specific module versions to be used, then
-            # override the environment variable to the deployed location on the remote host
-            f.write('source $NRP_MUSIC_DIRECTORY/nrp-variables\n')
-            f.write('export NRP_VARIABLES_PATH=$NRP_MUSIC_DIRECTORY/nrp-variables\n')
-
-            # load the environment modules based on the above configuration
-            proj_path = '/gpfs/bbp.cscs.ch/project/proj30/neurorobotics/%s/' % environment
-            f.write('source %s/server-scripts/nrp-services-modules.sh\n' % proj_path)
-
-            # set paths to models/experiments directory on gpfs
-            f.write('export NRP_MODELS_DIRECTORY=%s/models\n' % proj_path)
-            f.write('export NRP_EXPERIMENTS_DIRECTORY=%s/experiments\n' % proj_path)
-
-            # set the PYTHONPATH to add NRP modules on gpfs
-            venv_path = '%s/platform_venv/lib/python2.7/site-packages' % proj_path
-            f.write('export PYTHONPATH=%s:$PYTHONPATH\n' % venv_path)
-
-            # configure ROS and source the ros_venv before launching
-            f.write('export ROS_MASTER_URI=%s\n' % ros_master_uri)
-            f.write('source $ROS_PYTHON_VENV/bin/activate\n')
-
-            # actually launch the module
-            f.write('python -m {module}\n'.format(module=module))
-
-        # ensure the script is executable
-        os.chmod(path, stat.S_IRWXU)
-
-        # return a relative path to the script, it's guaranteed to be run in the tmpdir
-        return './%s' % name
-
-    def deploy(self):
-        """
-        Copy all configuration files to a temp directory on the remote host. The remote directory
-        is created here, so use it to set the launcher interface host tmpdir value.
-        """
-
-        # generated configuration files
-        for f in os.listdir(self._local_tmpdir):
-            self._copy_to_remote(os.path.join(self._local_tmpdir, f))
-
-        # deploy any NRP specific variable/module version configuration
-        self._copy_to_remote(os.environ.get('NRP_VARIABLES_PATH'))
-
-        # set the host tmpdir, slightly different naming convention between interfaces
-        self._host_tmpdir = self._tmp_dir
-
-    def shutdown(self):
-        """
-        Shutdown by trying to kill any running processes and deleting the temporary directory on the
-        remote allocated node and localhost. Both are guaranteed to exist by the constructor.
-        """
-
-        # try to terminate any processes running on the allocated node, they should already be
-        # terminated before we get here, catch any Exceptions and ensure we cleanup below
-        if self._host_tmpdir:
-            try:
-                # terminate any processes, remote directory will be deleted by parent cleanup
-                clean_process = self._spawn_ssh_node()
-                clean_process.sendline('for L in %s/*.lock ; do kill -9 `basename $L .lock`; done' %
-                                       self._host_tmpdir)
-                clean_process.terminate()
-            except Exception: # pylint: disable=broad-except
-                pass
-            finally:
-                self._host_tmpdir = None
-
-        # delete the remote temp directory and deallocate the node
-        LuganoVizCluster.stop(self)
-
-        # cleanup any local processes and delete the remote temp directory
-        LocalLauncher.shutdown(self)

hbp_nrp_music_interface/hbp_nrp_music_interface/launch/host/__init__.py

-"""
-This package contains host specific implementations for distributed MUSIC targets.
-"""