Merged in NRRPLT-7445 (pull request #16)

[NRRPLT-7445] Implements the import_experiment() function

* [NRRPLT-7445] Implements the import_experiment() function

* [NRRPLT-7445] Fixes pylint and pep8 errors

* [NRRPLT-7445] Fixes remaining pep8 error in it.py

* [NRRPLT-7445] Adds missing folder required by the unit tests

* [NRRPLT-7445] Adds the missing value experiment-import to the key proxy-services of the Config object

* [NRRPLT-7445] Addresses code review comments: uses request.codes.ok instead of httplib.OK

Approved-by: Manos Angelidis <angelidis@fortiss.org>

bitbucket-pipelines.yml

@@ -29,7 +29,7 @@ pipelines:
           - cp -af $HBP/user-scripts/config_files/platform_venv/* $VIRTUAL_ENV_PATH/lib/python2.7/site-packages/
 
           # Build
-          - export IGNORE_LINT='platform_venv|config_files'
+          - export IGNORE_LINT='platform_venv|config_files|examples/integration_test/test_experiment_folder'
           - make verify_base || { if [ -f pylint.txt ]; then echo "----------"; echo "PYLINT.TXT"; echo "----------";cat pylint.txt; fi; if [ -f pep8.txt ]; then echo "----------"; echo "PEP8.TXT"; echo "----------";cat pep8.txt; fi; exit 1; }
 
           # Coverage check

examples/integration_test/it.py

@@ -39,12 +39,12 @@ try:
 
 except ImportError:
     print
-    print 'Failed to import the VirtualCoach or access packages in the platform_venv! Aborting.'
+    print('Failed to import the VirtualCoach or access packages in the platform_venv! Aborting.')
     print
-    print 'Please make sure you have:'
+    print('Please make sure you have:')
     print
-    print '\t1. run "make devinstall" in the VirtualCoach repo'
-    print '\t2. use "cle-virtual-coach it.py" to run this script'
+    print('\t1. run "make devinstall" in the VirtualCoach repo')
+    print('\t2. use "cle-virtual-coach it.py" to run this script')
     print
     sys.exit(-1)
 
@@ -110,7 +110,7 @@ def run(oidc_username, storage_username):
 
     # running array of test case results, unfortunately we have to hardcode the number of
     # test cases because the indeterminate progress bar is not helpful for the tester
-    NUM_TEST_CASES = 28
+    NUM_TEST_CASES = 30
     results = TestCases(NUM_TEST_CASES)
 
     try:
@@ -174,6 +174,32 @@ def run(oidc_username, storage_username):
             raise TestCaseError('Deleting a cloned Experiment failed')
         results.done(True)
 
+        ##
+        ## Import an Experiment Folder
+        ##
+
+        results.start('Importing an experiment folder')
+        response = vc.import_experiment('test_experiment_folder')
+        new_experiment_id = json.loads(response.text)['destFolderName']
+        if new_experiment_id not in vc._VirtualCoach__get_experiment_list(cloned=True):
+            raise TestCaseError('Importing an experiment folder failed')
+        else:
+            vc.delete_cloned_experiment(new_experiment_id)
+        results.done(True)
+
+        ##
+        ## Import an Experiment Zipped Folder
+        ##
+
+        results.start('Importing a zipped experiment folder')
+        response = vc.import_experiment('test_experiment_folder.zip')
+        new_experiment_id = json.loads(response.text)['destFolderName']
+        if new_experiment_id not in vc._VirtualCoach__get_experiment_list(cloned=True):
+            raise TestCaseError('Importing an experiment folder failed')
+        else:
+            vc.delete_cloned_experiment(new_experiment_id)
+        results.done(True)
+
         ##
         ## Experiment Launch and Simulation State Interaction
         ##

examples/integration_test/test_experiment_folder/ExDTemplateHusky.exc

+<?xml version="1.0" encoding="utf-8"?>
+<ExD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xmlns="http://schemas.humanbrainproject.eu/SP10/2014/ExDConfig"
+     xsi:schemaLocation="http://schemas.humanbrainproject.eu/SP10/2014/ExDConfig ../ExDConfFile.xsd">
+    <name>Template Husky in empty environment</name>
+    <thumbnail>ExDTemplateHusky.jpg</thumbnail>
+    <description>This experiment loads the Husky robot in an empty world, with an idle brain and basic transfer functions. You are free to edit it.</description>
+    <tags>template husky robotics empty</tags>
+    <timeout>840</timeout>
+    <configuration type="3d-settings" src="ExDTemplateHusky.ini"/>
+    <configuration type="brainvisualizer" src="brainvisualizer.json"/>
+	<configuration type="user-interaction-settings" src="ExDTemplateHusky.uis"/>
+    <maturity>production</maturity>
+    <environmentModel src="empty_world/empty_world.sdf">
+        <robotPose robotId="husky" x="0.0" y="0.0" z="0.5" roll="0.0" pitch="-0.0" yaw="0.0"/>
+    </environmentModel>
+    <bibiConf src="template_husky.bibi"/>
+    <cameraPose>
+        <cameraPosition x="4.5" y="0" z="1.8"/>
+        <cameraLookAt x="0" y="0" z="0.6"/>
+    </cameraPose>
+</ExD>

examples/integration_test/test_experiment_folder/ExDTemplateHusky.ini

+{"dynamicEnvMap":true,"shadows":true,"antiAliasing":true,"ssao":false,"ssaoDisplay":false,"ssaoClamp":"0.94","ssaoLumInfluence":"1","rgbCurve":{"red":[[0,0],[0.410400390625,0.4481201171875],[0.761962890625,0.7957763671875],[1,1]],"green":[[0,0],[0.261962890625,0.2801513671875],[0.718994140625,0.7371826171875],[1,1]],"blue":[[0,0],[0.265869140625,0.2606201171875],[0.840087890625,0.7645263671875],[1,1]]},"levelsInBlack":"0.09","levelsInGamma":0.52,"levelsInWhite":"0.8","levelsOutBlack":0,"levelsOutWhite":1,"skyBox":"softgradient","sun":"","bloom":false,"bloomStrength":"0.83","bloomRadius":"0.29","bloomThreshold":"0.91","fog":true,"fogDensity":"0.02","fogColor":"#97a2af","pbrMaterial":true,"shadowSettings":[{"lightName":"sun","mapSize":4096,"cameraBottom":-10,"cameraLeft":-10,"cameraRight":10,"cameraTop":25,"bias":0.0003,"radius":1.2}]}
\ No newline at end of file

examples/integration_test/test_experiment_folder/ExDTemplateHusky.uis

+{
+    "camera": {
+        "defaultMode": "free-camera",
+        "sensitivity": {
+            "translation": 1.0,
+            "rotation": 1.0
+        }
+    }
+}

examples/integration_test/test_experiment_folder/all_neurons_spike_monitor.py

+# Imported Python Transfer Function
+#
+import hbp_nrp_cle.tf_framework as nrp
+# This specifies that the neurons 0 to 2 of the circuit population
+# should be monitored. You can see them in the spike train widget
+@nrp.NeuronMonitor(nrp.brain.record, nrp.spike_recorder)
+def all_neurons_spike_monitor(t):
+    # Uncomment to log into the 'log-console' visible in the simulation
+    # clientLogger.info("Time: ", t)
+    return True
+#
+

examples/integration_test/test_experiment_folder/brainvisualizer.json

+{}
\ No newline at end of file

examples/integration_test/test_experiment_folder/grab_image.py

+# Imported Python Transfer Function
+#
+import sensor_msgs.msg
+@nrp.MapRobotSubscriber("camera", Topic('/husky/husky/camera', sensor_msgs.msg.Image))
+@nrp.MapSpikeSource("input_neuron", nrp.brain.neurons[0], nrp.poisson)
+@nrp.Robot2Neuron()
+# Example TF: get image and fire at constant rate. You could do something with the image here and fire accordingly.
+def grab_image(t, camera, input_neuron):
+    image = camera.value
+    input_neuron.rate = 10
+#
+

examples/integration_test/test_experiment_folder/template_husky.bibi

+<?xml version="1.0" ?>
+<ns1:bibi xmlns:ns1="http://schemas.humanbrainproject.eu/SP10/2014/BIBI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<ns1:brainModel>
+		<ns1:file>brain_model/idle_brain.py</ns1:file>
+		<ns1:populations from="0" population="neurons" to="2" xsi:type="ns1:Range"/>
+		<ns1:populations from="0" population="record" to="2" xsi:type="ns1:Range"/>
+	</ns1:brainModel>
+	<ns1:bodyModel robotId="husky">husky_model/model.sdf</ns1:bodyModel>
+	<ns1:transferFunction src="all_neurons_spike_monitor.py" xsi:type="ns1:PythonTransferFunction"/>
+	<ns1:transferFunction src="turn_around.py" xsi:type="ns1:PythonTransferFunction"/>
+	<ns1:transferFunction src="grab_image.py" xsi:type="ns1:PythonTransferFunction"/>
+</ns1:bibi>

examples/integration_test/test_experiment_folder/turn_around.py

+# Imported Python Transfer Function
+#
+import hbp_nrp_cle.tf_framework as nrp
+from hbp_nrp_cle.robotsim.RobotInterface import Topic
+import geometry_msgs.msg
+@nrp.MapSpikeSink("output_neuron", nrp.brain.neurons[1], nrp.leaky_integrator_alpha)
+@nrp.Neuron2Robot(Topic('/husky/husky/cmd_vel', geometry_msgs.msg.Twist))
+# Example TF: get output neuron voltage and output constant on robot actuator. You could do something with the voltage here and command the robot accordingly.
+def turn_around(t, output_neuron):
+    voltage=output_neuron.voltage
+    return geometry_msgs.msg.Twist(linear=geometry_msgs.msg.Vector3(0,0,0),
+                                   angular=geometry_msgs.msg.Vector3(0,0,5))
+#
+

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py

@@ -86,7 +86,7 @@ class Config(dict):
         self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
                                            'experiment-clone', 'experiment-delete',
                                            'storage-authentication', 'storage-experiment-list',
-                                           'csv-files', 'experiment-file'])
+                                           'csv-files', 'experiment-file', 'experiment-import'])
         self.__validate('simulation-services', ['create', 'state', 'reset'])
         self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
                                                'sdf-world'])

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py

@@ -39,6 +39,7 @@ import copy
 from dateutil import parser
 import json
 from StringIO import StringIO
+import os
 
 
 class TestVirtualCoach(unittest.TestCase):
@@ -58,6 +59,7 @@ class TestVirtualCoach(unittest.TestCase):
             return realimport(name, globals, locals, fromlist, level)
         builtins.__import__ = rospy_import_fail
 
+        self._tests_path = os.path.dirname(os.path.realpath(__file__))
 
         self._vc = VirtualCoach(environment='local', storage_username='nrpuser')
 
@@ -626,3 +628,18 @@ mock-server-5
         request.return_value = Request()
         self._vc.set_experiment_file('exp_id', 'file_name', 'file_content')
 
+    @patch('requests.post')
+    def test_import_experiment_zip(self, mock_request):
+        mock_request.side_effect = [MagicMock(status_code=requests.codes.ok)]
+        self.assertRaises(Exception, self._vc.import_experiment, 'imaginary_file.zip')
+        path = os.path.join(self._tests_path, 'test.zip')
+        response = self._vc.import_experiment(path)
+        self.assertEqual(response.status_code, requests.codes.ok)
+
+    @patch('requests.post')
+    def test_import_experiment_folder(self, mock_request):
+        mock_request.side_effect = [MagicMock(status_code=requests.codes.ok)]
+        self.assertRaises(Exception, self._vc.import_experiment, 'imaginary_experiment_folder')
+        path = os.path.join(self._tests_path, 'test_experiment_folder')
+        response = self._vc.import_experiment(path)
+        self.assertEqual(response.status_code, requests.codes.ok)

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py

@@ -38,6 +38,9 @@ import requests
 import logging
 import getpass
 import httplib
+import os
+import zipfile
+import tempfile
 from texttable import Texttable
 from copy import copy
 
@@ -603,3 +606,74 @@ class VirtualCoach(object):
                             % (response.status_code, response))
 
         return response.content
+
+    @staticmethod
+    def __zip_directory(dirpath, zip_filehandle):
+        """
+        Internal helper function
+        It zips the target directory
+
+        :param dirpath: Path to the experiment folder to be zipped
+        :param zip_filehandle: Handle of the zip file to be populated
+        """
+        dirpath = os.path.abspath(dirpath)
+        dirname = os.path.dirname(dirpath)
+        basename = os.path.basename(dirpath)
+        os.chdir(dirname)
+        for root, _, files in os.walk(basename):
+            for f in files:
+                zip_filehandle.write(os.path.join(root, f))
+
+    @staticmethod
+    def __get_directory_content(dirpath):
+        """
+        Internal helper function
+        It zips the target folder and returns its content
+
+        :param dirpath: path to the experiment folder to be zipped
+        """
+        temp = tempfile.mktemp()
+        zip_file = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)
+        try:
+            VirtualCoach.__zip_directory(dirpath, zip_file)
+        except Exception as e:
+            logger.error('The folder %s could not be zipped', dirpath)
+            raise e
+        zip_file.close()
+        content = open(temp, 'r').read()
+        os.remove(temp)
+        return content
+
+    def import_experiment(self, path):
+        """
+        Imports an experiment folder, possibly a zipped folder, into user storage
+
+        :param path: path to the experiment folder or to the zip file to be imported
+        :type path: str
+        """
+        if not isinstance(path, str):
+            raise TypeError('The provided argument is not a string.')
+        if not os.path.isfile(path) and not os.path.isdir(path):
+            raise ValueError('The file or folder named %(path)s does not exist.' % {'path': path})
+
+        url = self.__config['proxy-services']['experiment-import']
+        file_headers = copy(self.__http_headers)
+        file_headers['Content-Type'] = 'application/octet-stream'
+        content = None
+        if os.path.isdir(path):
+            # Handles an experiment folder
+            content = VirtualCoach.__get_directory_content(path)
+        else:
+            # Handles a zip file
+            try:
+                content = open(path, 'r').read()
+            except Exception as e:
+                logger.error('The file %s could not be open', path)
+                raise e
+
+        response = requests.post(url, data=content, headers=file_headers)
+        # pylint: disable=no-member
+        if response.status_code != requests.codes.ok:
+            raise Exception('Error when importing experiment: %d. Error: %s'
+                            % (response.status_code, response))
+        return response

verify.sh

 #!/bin/bash
 # This script is designed for local usage.
 
-# Ignore generated files.
-export IGNORE_LINT="platform_venv"
+# Ignore generated files and files provided for testing.
+export IGNORE_LINT="platform_venv|config_files|examples/integration_test/test_experiment_folder"
 export VIRTUAL_ENV=$NRP_VIRTUAL_ENV
 
 # This script only runs static code analysis, the tests can be run separately using run_tests.sh