From 502a7488e8071bb641d0343af2a14b68e483421a Mon Sep 17 00:00:00 2001
From: Claudio Sousa <claudio.sousa@gmail.com>
Date: Thu, 14 Mar 2019 13:21:24 +0000
Subject: [PATCH] Merged in NRRPLT-7268_csv_api (pull request #10)
[NRRPLT-7268] VC uses the new proxy call to retrieve CSVs
* [NRRPLT-7268] VC uses the new proxy call to retrieve CSVs
* [NRRPLT-7268] missing config validation
* [NRRPLT-7268] returning csv content as string
Approved-by: Manos Angelidis <angelidis@fortiss.org>
Approved-by: Luc Guyot <luc.guyot@epfl.ch>
---
.../hbp_nrp_virtual_coach/config.py | 3 +-
.../hbp_nrp_virtual_coach/simulation.py | 12 +-
.../tests/test_simulation.py | 20 ++-
.../tests/test_virtual_coach.py | 108 ++++++++++++-
.../hbp_nrp_virtual_coach/virtual_coach.py | 148 +++++++++++++++++-
5 files changed, 275 insertions(+), 16 deletions(-)
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
index 7295a03..53d8425 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
@@ -85,7 +85,8 @@ class Config(dict):
self.__validate('proxy', ['staging', 'dev', 'local', environment])
self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
'experiment-clone', 'experiment-delete',
- 'storage-authentication', 'storage-experiment-list'])
+ 'storage-authentication', 'storage-experiment-list',
+ 'csv-files', 'experiment-file'])
self.__validate('simulation-services', ['create', 'state', 'reset'])
self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
'sdf-world'])
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
index ab56251..8a6152f 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
@@ -40,18 +40,20 @@ class Simulation(object):
Provides an interface to launch or control a simulation instance.
"""
- def __init__(self, http_client, config):
+ def __init__(self, http_client, config, vc):
"""
Initialize a simulation interface and default logger.
:param http_client: A HTTP client.
:param config: A loaded Virtual Coach config.
+ :param vc: The virtual coach instance.
"""
assert isinstance(config, Config)
assert isinstance(http_client, HTTPClient)
self.__http_client = http_client
self.__config = config
+ self.__vc = vc
self.__server = None
self.__server_info = None
@@ -857,3 +859,11 @@ class Simulation(object):
"Unable to reset simulation, HTTP status %s" % status_code)
self.__logger.info('Reset completed. The simulation has been paused and will not be started'
' automatically.')
+
+ def get_last_run_csv_file(self, file_name):
+ """
+ Retrieves a csv file content for the last simulation run.
+
+ :param file_name: The name of csv file for which to retrieve the content.
+ """
+ return self.__vc.get_last_run_csv_file(self.__experiment_id, file_name)
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
index f51d830..2bf450f 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
@@ -39,13 +39,10 @@ import json
from StringIO import StringIO
-
class TestSimulation(unittest.TestCase):
-
def setUp(self):
- self._sim = Simulation(RequestsClient({'Authorization': 'token'}), Config('local'))
-
+ self._sim = Simulation(RequestsClient({'Authorization': 'token'}), Config('local'), None)
def setUpForLaunch(self):
get_response = (None,
@@ -56,8 +53,8 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__http_client.post = Mock(return_value=post_response)
def test_init_asserts(self):
- self.assertRaises(AssertionError, Simulation, None, Config('local'))
- self.assertRaises(AssertionError, Simulation, RequestsClient({'Authorization': 'token'}), None)
+ self.assertRaises(AssertionError, Simulation, None, Config('local'), None)
+ self.assertRaises(AssertionError, Simulation, RequestsClient({'Authorization': 'token'}), None, None)
def test_launch_asserts(self):
self.assertRaises(AssertionError, self._sim.launch, None, 'conf', 'server', None)
@@ -77,7 +74,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__http_client.get.assert_called_once()
self._sim._Simulation__http_client.post.assert_not_called()
mocked_traceback.print_exec.assert_called_once()
-
+
@patch('hbp_nrp_virtual_coach.simulation.traceback')
def test_failed_create_conflict(self, mocked_traceback):
self.setUpForLaunch()
@@ -476,7 +473,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__headers = {}
self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK, None))
-
+
self._sim.save_transfer_functions()
self._sim._Simulation__http_client.put.assert_called_once(
@@ -634,3 +631,10 @@ class TestSimulation(unittest.TestCase):
self.assertRaises(ValueError, self._sim.reset, 'foo')
self.assertRaises(Exception, self._sim.reset, 'full')
self._sim.start.assert_called_once()
+
+ def test_get_csv_last_run_file(self):
+ self._sim._Simulation__vc = Mock()
+ self._sim._Simulation__vc.get_last_run_csv_file = Mock()
+ self._sim._Simulation__experiment_id = 'experiment_id'
+ self._sim.get_last_run_csv_file('file_name')
+ self._sim._Simulation__vc.get_last_run_csv_file.assert_called_once_with('experiment_id', 'file_name')
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
index 9b18051..9f25da1 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
@@ -29,7 +29,7 @@ from hbp_nrp_virtual_coach.virtual_coach import VirtualCoach
from bbp_client.oidc.client import BBPOIDCClient
-from mock import Mock, patch
+from mock import Mock, patch, MagicMock
import unittest
import requests
import getpass
@@ -57,7 +57,7 @@ class TestVirtualCoach(unittest.TestCase):
raise ImportError('no ROS for tests')
return realimport(name, globals, locals, fromlist, level)
builtins.__import__ = rospy_import_fail
-
+
self._vc = VirtualCoach(environment='local', storage_username='nrpuser')
@@ -134,14 +134,14 @@ class TestVirtualCoach(unittest.TestCase):
def test_no_login_credentials(self):
self.assertRaises(Exception, VirtualCoach)
-
+
@patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_storage_token')
@patch('getpass.getpass', return_value='password')
def test_storage_auth(self, mock_getpass, mock_storage_login):
# mocked storage authentication, ensure called if username provided
VirtualCoach(environment='local', storage_username='nrpuser')
mock_storage_login.assert_called_once_with('nrpuser', 'password')
-
+
@patch('bbp_client.oidc.client.BBPOIDCClient.implicit_auth')
def test_oidc_auth(self, mock_oidc_login):
# mocked OIDC authentication, ensure called if username provided
@@ -453,7 +453,7 @@ mock-server-5
request.return_value = Request()
self._vc._VirtualCoach__storage_username = None
self.assertRaises(ValueError, self._vc.clone_cloned_experiment, 'exp_id')
-
+
vc_oidc_username = VirtualCoach(environment='dev', oidc_username='youknowwho')
vc_oidc_username._VirtualCoach__http_client._OIDCHTTPClient__oidc_client.request = Mock(return_value=[{'status': 477}, 'something'])
self.assertRaises(Exception, vc_oidc_username.clone_cloned_experiment, 'exp_id')
@@ -511,3 +511,101 @@ mock-server-5
mock_request.return_value = Response()
content = self._vc._VirtualCoach__get_storage_token('user', 'pass')
self.assertEqual(content, 'token')
+
+ def __mock_csv_files_response(self):
+ class Response(object):
+ status_code = 200
+ content = """
+ [
+ { "name": "csv1", "folder": "folder1", "size":3, "uuid": "uuid1" },
+ { "name": "csv2", "folder": "folder1", "size":2, "uuid": "uuid2" },
+ { "name": "csv3", "folder": "folder1", "size":4, "uuid": "uuid3" },
+ { "name": "csv1", "folder": "folder2", "size":5, "uuid": "uuid4" }
+ ]
+ """
+
+ return Response()
+
+ @patch('requests.get')
+ @patch('sys.stdout', new_callable=StringIO)
+ def test_print_runs(self, mock_stdout,mock_request):
+ mock_request.return_value = self.__mock_csv_files_response()
+
+ content = self._vc.print_runs('exp_id')
+ csv_runs = """
++--------+---------+-------+
+| Run id | Date | Bytes |
++========+=========+=======+
+| 0 | folder1 | 9 |
++--------+---------+-------+
+| 1 | folder2 | 5 |
++--------+---------+-------+
+ """
+ self.assertEqual(mock_stdout.getvalue().strip(), csv_runs.strip())
+
+ @patch('requests.get')
+ @patch('sys.stdout', new_callable=StringIO)
+ def test_print_csv_run_files(self, mock_stdout,mock_request):
+ mock_request.return_value = self.__mock_csv_files_response()
+
+ self._vc.print_run_csv_files('exp_id', 0)
+ csv_run_files = """
++------+------+
+| File | Size |
++======+======+
+| csv1 | 3 |
++------+------+
+| csv2 | 2 |
++------+------+
+| csv3 | 4 |
++------+------+
+"""
+ self.assertEqual(mock_stdout.getvalue().strip(), csv_run_files.strip())
+
+ @patch('requests.get')
+ @patch('sys.stdout', new_callable=StringIO)
+ def test_print_csv_last_run_files(self, mock_stdout,mock_request):
+ mock_request.return_value = self.__mock_csv_files_response()
+
+ self._vc.print_last_run_csv_files('exp_id')
+ csv_run_files = """
++------+------+
+| File | Size |
++======+======+
+| csv1 | 5 |
++------+------+
+"""
+ self.assertEqual(mock_stdout.getvalue().strip(), csv_run_files.strip())
+
+ def __mock_csv_file_response(self):
+ class Response(object):
+ status_code = 200
+ content ='a,b,c\n1,2,3'
+
+ return Response()
+
+ @patch('requests.get')
+ def test_get_run_csv_file(self, mock_request):
+ mock_request.side_effect = [
+ self.__mock_csv_files_response(),
+ self.__mock_csv_file_response()
+ ]
+ self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
+ csv_content = self._vc.get_run_csv_file('exp_id', 0, 'csv1')
+
+ self.assertEqual(csv_content, 'a,b,c\n1,2,3')
+ self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid1')
+
+ @patch('requests.get')
+ def test_get_last_run_csv_file(self, mock_request):
+ mock_request.side_effect = [
+ self.__mock_csv_files_response(),
+ self.__mock_csv_file_response()
+ ]
+
+ self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
+ csv_content = self._vc.get_last_run_csv_file('exp_id', 'csv1')
+ self.assertEqual(csv_content, 'a,b,c\n1,2,3')
+ self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid4')
+
+
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
index af88278..221be57 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
@@ -31,6 +31,7 @@ from hbp_nrp_virtual_coach.requests_client import RequestsClient
from hbp_nrp_virtual_coach.oidc_http_client import OIDCHTTPClient
from datetime import datetime, timedelta
+from collections import defaultdict
from dateutil import parser, tz
import json
import requests
@@ -279,7 +280,7 @@ class VirtualCoach(object):
# attempt to launch the simulation on all server targets, on success return an interface
# to the simulation
- sim = Simulation(self.__http_client, self.__config)
+ sim = Simulation(self.__http_client, self.__config, self)
for server in servers:
try:
if sim.launch(experiment_id, str(experiment_conf), str(server),
@@ -427,3 +428,148 @@ class VirtualCoach(object):
raise Exception('Error when getting server list, Status Code: %d. Error: %s'
% (status_code, response))
return json.loads(response)
+
+ def __get_available_CSV_files(self, experiment_id):
+ """
+ Internal helper to retrieve the list of CSV files available for an experiment
+
+ :param experiment_id: The experiment id for which to retrieve the list of CSV files
+ """
+ response = requests.get(self.__config['proxy-services']['csv-files'] % (experiment_id,),
+ headers=self.__http_headers)
+
+ if response.status_code != httplib.OK:
+ raise Exception('Error when getting CSV files Status Code: %d. Error: %s'
+ % (response.status_code, response))
+ csv_files = json.loads(response.content)
+ distinct_runs = defaultdict(dict)
+ for csv_file in csv_files:
+ distinct_runs[csv_file['folder']][csv_file['name']] = csv_file
+ return distinct_runs
+
+ def print_runs(self, exp_id):
+ """
+ Prints the list of simulation runs that generated CSV files
+
+ :param exp_id: The experiment id for which to retrieve the list of CSV simulation runs
+ """
+ csv_files = self.__get_available_CSV_files(exp_id)
+
+ table = Texttable()
+ table.header(['Run id', 'Date', 'Bytes'])
+ table.set_cols_align(['r', 'c', 'r'])
+
+ for i, run_date in enumerate(sorted(csv_files.keys())):
+ run_size = sum(file['size'] for file in csv_files[run_date].values())
+ table.add_row([i, run_date, run_size])
+
+ logger.info('List of simulation runs')
+ print table.draw()
+
+ def print_run_csv_files(self, exp_id, run_id):
+ """
+ Prints the list of CSV files for a given run
+
+ :param exp_id: The experiment id for which to retrieve the list of CSV files
+ :param run_id: The run id for which to retrieve the list of CSV files
+ """
+ csv_files = self.__get_available_CSV_files(exp_id)
+
+ table = Texttable()
+ table.header(['File', 'Size'])
+ table.set_cols_align(['l', 'r'])
+
+ sorted_runs = sorted(csv_files.keys())
+ if not 0 <= run_id < len(sorted_runs):
+ raise Exception('Could not find run %i, %i runs were found' %
+ (run_id, len(sorted_runs)))
+
+ for csv_file in csv_files[sorted_runs[run_id]].values():
+ table.add_row([csv_file['name'], csv_file['size']])
+
+ logger.info('Run %i list of files.', run_id)
+ print table.draw()
+
+ def print_last_run_csv_files(self, exp_id):
+ """
+ Prints the list of CSV files for the last run
+
+ :param exp_id: The experiment id for which to retrieve the list of CSV files
+ """
+ csv_files = self.__get_available_CSV_files(exp_id)
+
+ table = Texttable()
+ table.header(['File', 'Size'])
+ table.set_cols_align(['l', 'r'])
+
+ sorted_runs = sorted(csv_files.keys())
+ if not sorted_runs:
+ raise Exception('Could not find any run')
+
+ for csv_file in csv_files[sorted_runs[-1]].values():
+ table.add_row([csv_file['name'], csv_file['size']])
+
+ logger.info('Last run list of files')
+ print table.draw()
+
+ def __get_csv_file_content(self, exp_id, file_uuid):
+ """
+ Internal helper method to retrieve a CSV file content
+
+ :param exp_id: The experiment id for which to retrieve the CSV file content
+ :param file_uuid: The file uuid for which to retrieve the content
+ """
+ logger.info('Retrieving CSV file.')
+
+ response = requests.get(self.__config['proxy-services']['experiment-file'] %
+ (exp_id, file_uuid),
+ headers=self.__http_headers)
+
+ if response.status_code != httplib.OK:
+ raise Exception('Error when getting CSV file Status Code: %d. Error: %s'
+ % (response.status_code, response))
+
+ return response.content
+
+ def get_run_csv_file(self, exp_id, run_id, file_name):
+ """
+ Retrieves a CSV file content
+
+ :param exp_id: The experiment id
+ :param run_id: The run id
+ :param file_uuid: The file uuid
+ """
+ csv_files = self.__get_available_CSV_files(exp_id)
+ sorted_runs = sorted(csv_files.keys())
+ if not 0 <= run_id < len(sorted_runs):
+ raise Exception('Could not find run %i, %i runs were found' %
+ (run_id, len(sorted_runs)))
+
+ if file_name not in csv_files[sorted_runs[run_id]]:
+ file_names = ', '.join(f['name'] for f in csv_files[sorted_runs[run_id]].values())
+ raise Exception('Could not find file \'%s\' in run %i, available file names are: %s' %
+ (file_name, run_id, file_names))
+
+ file_uuid = csv_files[sorted_runs[run_id]][file_name]['uuid']
+ return self.__get_csv_file_content(exp_id, file_uuid)
+
+ def get_last_run_csv_file(self, exp_id, file_name):
+ """
+ Retrieves a CSV file content for the last run
+
+ :param exp_id: The experiment id
+ :param file_name: The file name
+ """
+
+ csv_files = self.__get_available_CSV_files(exp_id)
+ sorted_runs = sorted(csv_files.keys())
+ if not sorted_runs:
+ raise Exception('Could not find any run')
+
+ if file_name not in csv_files[sorted_runs[-1]]:
+ file_names = ', '.join(file['name'] for file in csv_files[sorted_runs[-1]].values())
+ raise Exception('Could not find file \'%s\' in last run, available file names are: %s' %
+ (file_name, file_names))
+
+ file_uuid = csv_files[sorted_runs[-1]][file_name]['uuid']
+ return self.__get_csv_file_content(exp_id, file_uuid)
--
GitLab