Merged in NRRPLT-7268_csv_api (pull request #10)

[NRRPLT-7268] VC uses the new proxy call to retrieve CSVs

* [NRRPLT-7268] VC uses the new proxy call to retrieve CSVs

* [NRRPLT-7268] missing config validation

* [NRRPLT-7268] returning csv content as string

Approved-by: Manos Angelidis <angelidis@fortiss.org>
Approved-by: Luc Guyot <luc.guyot@epfl.ch>

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py

@@ -85,7 +85,8 @@ class Config(dict):
         self.__validate('proxy', ['staging', 'dev', 'local', environment])
         self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
                                            'experiment-clone', 'experiment-delete',
-                                           'storage-authentication', 'storage-experiment-list'])
+                                           'storage-authentication', 'storage-experiment-list',
+                                           'csv-files', 'experiment-file'])
         self.__validate('simulation-services', ['create', 'state', 'reset'])
         self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
                                                'sdf-world'])

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py

@@ -40,18 +40,20 @@ class Simulation(object):
     Provides an interface to launch or control a simulation instance.
     """
 
-    def __init__(self, http_client, config):
+    def __init__(self, http_client, config, vc):
         """
         Initialize a simulation interface and default logger.
 
         :param http_client: A HTTP client.
         :param config: A loaded Virtual Coach config.
+        :param vc: The virtual coach instance.
         """
         assert isinstance(config, Config)
         assert isinstance(http_client, HTTPClient)
 
         self.__http_client = http_client
         self.__config = config
+        self.__vc = vc
 
         self.__server = None
         self.__server_info = None
@@ -857,3 +859,11 @@ class Simulation(object):
                 "Unable to reset simulation, HTTP status %s" % status_code)
         self.__logger.info('Reset completed. The simulation has been paused and will not be started'
                            ' automatically.')
+
+    def get_last_run_csv_file(self, file_name):
+        """
+        Retrieves a csv file content for the last simulation run.
+
+        :param file_name: The name of csv file for which to retrieve the content.
+        """
+        return self.__vc.get_last_run_csv_file(self.__experiment_id, file_name)

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py

@@ -39,13 +39,10 @@ import json
 from StringIO import StringIO
 
 
-
 class TestSimulation(unittest.TestCase):
 
-    
     def setUp(self):
-        self._sim = Simulation(RequestsClient({'Authorization': 'token'}), Config('local'))
-
+        self._sim = Simulation(RequestsClient({'Authorization': 'token'}), Config('local'), None)
 
     def setUpForLaunch(self):
         get_response = (None,
@@ -56,8 +53,8 @@ class TestSimulation(unittest.TestCase):
         self._sim._Simulation__http_client.post = Mock(return_value=post_response)
 
     def test_init_asserts(self):
-        self.assertRaises(AssertionError, Simulation, None, Config('local'))
-        self.assertRaises(AssertionError, Simulation, RequestsClient({'Authorization': 'token'}), None)
+        self.assertRaises(AssertionError, Simulation, None, Config('local'), None)
+        self.assertRaises(AssertionError, Simulation, RequestsClient({'Authorization': 'token'}), None, None)
 
     def test_launch_asserts(self):
         self.assertRaises(AssertionError, self._sim.launch, None, 'conf', 'server', None)
@@ -77,7 +74,7 @@ class TestSimulation(unittest.TestCase):
         self._sim._Simulation__http_client.get.assert_called_once()
         self._sim._Simulation__http_client.post.assert_not_called()
         mocked_traceback.print_exec.assert_called_once()
-    
+
     @patch('hbp_nrp_virtual_coach.simulation.traceback')
     def test_failed_create_conflict(self, mocked_traceback):
         self.setUpForLaunch()
@@ -476,7 +473,7 @@ class TestSimulation(unittest.TestCase):
         self._sim._Simulation__headers = {}
 
         self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK, None))
- 
+
 
         self._sim.save_transfer_functions()
         self._sim._Simulation__http_client.put.assert_called_once(
@@ -634,3 +631,10 @@ class TestSimulation(unittest.TestCase):
         self.assertRaises(ValueError, self._sim.reset, 'foo')
         self.assertRaises(Exception, self._sim.reset, 'full')
         self._sim.start.assert_called_once()
+
+    def test_get_csv_last_run_file(self):
+        self._sim._Simulation__vc = Mock()
+        self._sim._Simulation__vc.get_last_run_csv_file = Mock()
+        self._sim._Simulation__experiment_id = 'experiment_id'
+        self._sim.get_last_run_csv_file('file_name')
+        self._sim._Simulation__vc.get_last_run_csv_file.assert_called_once_with('experiment_id', 'file_name')

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py

@@ -29,7 +29,7 @@ from hbp_nrp_virtual_coach.virtual_coach import VirtualCoach
 
 from bbp_client.oidc.client import BBPOIDCClient
 
-from mock import Mock, patch
+from mock import Mock, patch, MagicMock
 import unittest
 import requests
 import getpass
@@ -57,7 +57,7 @@ class TestVirtualCoach(unittest.TestCase):
                 raise ImportError('no ROS for tests')
             return realimport(name, globals, locals, fromlist, level)
         builtins.__import__ = rospy_import_fail
-        
+
 
         self._vc = VirtualCoach(environment='local', storage_username='nrpuser')
 
@@ -134,14 +134,14 @@ class TestVirtualCoach(unittest.TestCase):
     def test_no_login_credentials(self):
         self.assertRaises(Exception, VirtualCoach)
 
-    
+
     @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_storage_token')
     @patch('getpass.getpass', return_value='password')
     def test_storage_auth(self, mock_getpass, mock_storage_login):
         # mocked storage authentication, ensure called if username provided
         VirtualCoach(environment='local', storage_username='nrpuser')
         mock_storage_login.assert_called_once_with('nrpuser', 'password')
-    
+
     @patch('bbp_client.oidc.client.BBPOIDCClient.implicit_auth')
     def test_oidc_auth(self, mock_oidc_login):
         # mocked OIDC authentication, ensure called if username provided
@@ -453,7 +453,7 @@ mock-server-5
         request.return_value = Request()
         self._vc._VirtualCoach__storage_username = None
         self.assertRaises(ValueError, self._vc.clone_cloned_experiment, 'exp_id')
-        
+
         vc_oidc_username = VirtualCoach(environment='dev', oidc_username='youknowwho')
         vc_oidc_username._VirtualCoach__http_client._OIDCHTTPClient__oidc_client.request = Mock(return_value=[{'status': 477}, 'something'])
         self.assertRaises(Exception, vc_oidc_username.clone_cloned_experiment, 'exp_id')
@@ -511,3 +511,101 @@ mock-server-5
         mock_request.return_value = Response()
         content = self._vc._VirtualCoach__get_storage_token('user', 'pass')
         self.assertEqual(content, 'token')
+
+    def __mock_csv_files_response(self):
+        class Response(object):
+            status_code = 200
+            content = """
+            [
+                { "name": "csv1", "folder": "folder1", "size":3, "uuid": "uuid1" },
+                { "name": "csv2", "folder": "folder1", "size":2, "uuid": "uuid2" },
+                { "name": "csv3", "folder": "folder1", "size":4, "uuid": "uuid3" },
+                { "name": "csv1", "folder": "folder2", "size":5, "uuid": "uuid4" }
+            ]
+            """
+
+        return Response()
+
+    @patch('requests.get')
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_print_runs(self, mock_stdout,mock_request):
+        mock_request.return_value  = self.__mock_csv_files_response()
+
+        content = self._vc.print_runs('exp_id')
+        csv_runs = """
++--------+---------+-------+
+| Run id |  Date   | Bytes |
++========+=========+=======+
+|      0 | folder1 |     9 |
++--------+---------+-------+
+|      1 | folder2 |     5 |
++--------+---------+-------+
+        """
+        self.assertEqual(mock_stdout.getvalue().strip(), csv_runs.strip())
+
+    @patch('requests.get')
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_print_csv_run_files(self, mock_stdout,mock_request):
+        mock_request.return_value  = self.__mock_csv_files_response()
+
+        self._vc.print_run_csv_files('exp_id', 0)
+        csv_run_files = """
++------+------+
+| File | Size |
++======+======+
+| csv1 |    3 |
++------+------+
+| csv2 |    2 |
++------+------+
+| csv3 |    4 |
++------+------+
+"""
+        self.assertEqual(mock_stdout.getvalue().strip(), csv_run_files.strip())
+
+    @patch('requests.get')
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_print_csv_last_run_files(self, mock_stdout,mock_request):
+        mock_request.return_value  = self.__mock_csv_files_response()
+
+        self._vc.print_last_run_csv_files('exp_id')
+        csv_run_files = """
++------+------+
+| File | Size |
++======+======+
+| csv1 |    5 |
++------+------+
+"""
+        self.assertEqual(mock_stdout.getvalue().strip(), csv_run_files.strip())
+
+    def __mock_csv_file_response(self):
+        class Response(object):
+            status_code = 200
+            content ='a,b,c\n1,2,3'
+
+        return Response()
+
+    @patch('requests.get')
+    def test_get_run_csv_file(self, mock_request):
+        mock_request.side_effect = [
+            self.__mock_csv_files_response(),
+            self.__mock_csv_file_response()
+        ]
+        self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
+        csv_content = self._vc.get_run_csv_file('exp_id', 0, 'csv1')
+
+        self.assertEqual(csv_content, 'a,b,c\n1,2,3')
+        self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid1')
+
+    @patch('requests.get')
+    def test_get_last_run_csv_file(self, mock_request):
+        mock_request.side_effect = [
+            self.__mock_csv_files_response(),
+            self.__mock_csv_file_response()
+        ]
+
+        self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
+        csv_content = self._vc.get_last_run_csv_file('exp_id', 'csv1')
+        self.assertEqual(csv_content, 'a,b,c\n1,2,3')
+        self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid4')
+
+

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py

@@ -31,6 +31,7 @@ from hbp_nrp_virtual_coach.requests_client import RequestsClient
 from hbp_nrp_virtual_coach.oidc_http_client import OIDCHTTPClient
 
 from datetime import datetime, timedelta
+from collections import defaultdict
 from dateutil import parser, tz
 import json
 import requests
@@ -279,7 +280,7 @@ class VirtualCoach(object):
 
         # attempt to launch the simulation on all server targets, on success return an interface
         # to the simulation
-        sim = Simulation(self.__http_client, self.__config)
+        sim = Simulation(self.__http_client, self.__config, self)
         for server in servers:
             try:
                 if sim.launch(experiment_id, str(experiment_conf), str(server),
@@ -427,3 +428,148 @@ class VirtualCoach(object):
             raise Exception('Error when getting server list, Status Code: %d. Error: %s'
                             % (status_code, response))
         return json.loads(response)
+
+    def __get_available_CSV_files(self, experiment_id):
+        """
+        Internal helper to retrieve the list of CSV files available for an experiment
+
+        :param experiment_id: The experiment id for which to retrieve the list of CSV files
+        """
+        response = requests.get(self.__config['proxy-services']['csv-files'] % (experiment_id,),
+                                headers=self.__http_headers)
+
+        if response.status_code != httplib.OK:
+            raise Exception('Error when getting CSV files Status Code: %d. Error: %s'
+                            % (response.status_code, response))
+        csv_files = json.loads(response.content)
+        distinct_runs = defaultdict(dict)
+        for csv_file in csv_files:
+            distinct_runs[csv_file['folder']][csv_file['name']] = csv_file
+        return distinct_runs
+
+    def print_runs(self, exp_id):
+        """
+        Prints the list of simulation runs that generated CSV files
+
+        :param exp_id: The experiment id for which to retrieve the list of CSV simulation runs
+        """
+        csv_files = self.__get_available_CSV_files(exp_id)
+
+        table = Texttable()
+        table.header(['Run id', 'Date', 'Bytes'])
+        table.set_cols_align(['r', 'c', 'r'])
+
+        for i, run_date in enumerate(sorted(csv_files.keys())):
+            run_size = sum(file['size'] for file in csv_files[run_date].values())
+            table.add_row([i, run_date, run_size])
+
+        logger.info('List of simulation runs')
+        print table.draw()
+
+    def print_run_csv_files(self, exp_id, run_id):
+        """
+        Prints the list of CSV files for a given run
+
+        :param exp_id: The experiment id for which to retrieve the list of CSV files
+        :param run_id: The run id for which to retrieve the list of CSV files
+        """
+        csv_files = self.__get_available_CSV_files(exp_id)
+
+        table = Texttable()
+        table.header(['File', 'Size'])
+        table.set_cols_align(['l', 'r'])
+
+        sorted_runs = sorted(csv_files.keys())
+        if not 0 <= run_id < len(sorted_runs):
+            raise Exception('Could not find run %i, %i runs were found' %
+                            (run_id, len(sorted_runs)))
+
+        for csv_file in csv_files[sorted_runs[run_id]].values():
+            table.add_row([csv_file['name'], csv_file['size']])
+
+        logger.info('Run %i list of files.', run_id)
+        print table.draw()
+
+    def print_last_run_csv_files(self, exp_id):
+        """
+        Prints the list of CSV files for the last run
+
+        :param exp_id: The experiment id for which to retrieve the list of CSV files
+        """
+        csv_files = self.__get_available_CSV_files(exp_id)
+
+        table = Texttable()
+        table.header(['File', 'Size'])
+        table.set_cols_align(['l', 'r'])
+
+        sorted_runs = sorted(csv_files.keys())
+        if not sorted_runs:
+            raise Exception('Could not find any run')
+
+        for csv_file in csv_files[sorted_runs[-1]].values():
+            table.add_row([csv_file['name'], csv_file['size']])
+
+        logger.info('Last run list of files')
+        print table.draw()
+
+    def __get_csv_file_content(self, exp_id, file_uuid):
+        """
+        Internal helper method to retrieve a CSV file content
+
+        :param exp_id: The experiment id for which to retrieve the CSV file content
+        :param file_uuid: The file uuid for which to retrieve the content
+        """
+        logger.info('Retrieving CSV file.')
+
+        response = requests.get(self.__config['proxy-services']['experiment-file'] %
+                                    (exp_id, file_uuid),
+                                headers=self.__http_headers)
+
+        if response.status_code != httplib.OK:
+            raise Exception('Error when getting CSV file Status Code: %d. Error: %s'
+                            % (response.status_code, response))
+
+        return response.content
+
+    def get_run_csv_file(self, exp_id, run_id, file_name):
+        """
+        Retrieves a CSV file content
+
+        :param exp_id: The experiment id
+        :param run_id: The run id
+        :param file_uuid: The file uuid
+        """
+        csv_files = self.__get_available_CSV_files(exp_id)
+        sorted_runs = sorted(csv_files.keys())
+        if not 0 <= run_id < len(sorted_runs):
+            raise Exception('Could not find run %i, %i runs were found' %
+                            (run_id, len(sorted_runs)))
+
+        if file_name not in csv_files[sorted_runs[run_id]]:
+            file_names = ', '.join(f['name'] for f in csv_files[sorted_runs[run_id]].values())
+            raise Exception('Could not find file \'%s\' in run %i, available file names are: %s' %
+                            (file_name, run_id, file_names))
+
+        file_uuid = csv_files[sorted_runs[run_id]][file_name]['uuid']
+        return self.__get_csv_file_content(exp_id, file_uuid)
+
+    def get_last_run_csv_file(self, exp_id, file_name):
+        """
+        Retrieves a CSV file content for the last run
+
+        :param exp_id: The experiment id
+        :param file_name: The file name
+        """
+
+        csv_files = self.__get_available_CSV_files(exp_id)
+        sorted_runs = sorted(csv_files.keys())
+        if not sorted_runs:
+            raise Exception('Could not find any run')
+
+        if file_name not in csv_files[sorted_runs[-1]]:
+            file_names = ', '.join(file['name'] for file in csv_files[sorted_runs[-1]].values())
+            raise Exception('Could not find file \'%s\' in last run, available file names are: %s' %
+                            (file_name, file_names))
+
+        file_uuid = csv_files[sorted_runs[-1]][file_name]['uuid']
+        return self.__get_csv_file_content(exp_id, file_uuid)