From e6c3bfa4a4111dfb637c3c5abe324b7b616a6fc9 Mon Sep 17 00:00:00 2001 From: claudio <claudio.sousa@epfl.ch> Date: Thu, 29 Mar 2018 14:16:18 +0200 Subject: [PATCH] [NRRPLT-6306] Save to storage Change-Id: I6a6de3c4ea3b603dbe2f19a23bb418e606258a61 --- .../hbp_nrp_virtual_coach/config.py | 3 +- .../hbp_nrp_virtual_coach/simulation.py | 102 +++++++++++++++++- .../tests/test_simulation.py | 101 +++++++++++++++++ .../tests/test_virtual_coach.py | 20 +++- 4 files changed, 221 insertions(+), 5 deletions(-) diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py index 10e2248..b6604f9 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py @@ -87,7 +87,8 @@ class Config(dict): 'experiment-clone', 'experiment-delete', 'storage-authentication', 'storage-experiment-list']) self.__validate('simulation-services', ['create', 'state', 'reset', 'csv-recorders']) - self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain']) + self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain', + 'sdf-world']) self.__validate('reset-services', ['robot_pose', 'full', 'world', 'brain']) self.__validate('ros', ['status', 'error']) diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py index 86628a3..219b6b4 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py @@ -526,10 +526,110 @@ class Simulation(object): Deletes a transfer function. :param transfer_function_name: A string containing the name of the transfer function to be - deleted. + deleted. """ self.__delete_script('transfer-function', transfer_function_name) + def __save_experiment_data(self, experiment_data_type, experiment_data, method='put'): + """ + Saves data related to the experiment. + Url format: in the format http://localhost:8080/experiment/{exp-id}/brain + + :param experiment_data_type: the type of experiment data to save (i.e. 'transfer-function', + 'state-machine', 'brain' or 'sdf-world') + :param experiment_data: the experiment data to be saved + :param method: the http request to be executed. Default is: PUT + """ + + url = '%s/experiment/%s/%s' % (self.__server_info['gzweb']['nrp-services'], + self.__experiment_id, + self.__config['simulation-scripts'][experiment_data_type]) + try: + method_fun = getattr(requests, method) + res = method_fun(url, headers=self.__storage_headers, json=experiment_data) + + if res.status_code != httplib.OK: + raise Exception('Error status code %s' % res.status_code) + self.__logger.info("Saved %s." % experiment_data_type) + except Exception as err: + self.__logger.info(err) + raise Exception("Failed to save %s." % experiment_data_type) + + def save_transfer_functions(self, transfer_functions_list): + """ + Saves the transfer functions in the storage + + :param transfer_functions_list: List[str] of strings where each string is the code of a + transfer function + """ + + assert isinstance(transfer_functions_list, list) + + return self.__save_experiment_data('transfer-function', { + 'experimentId': self.__experiment_id, + 'transfer_functions': transfer_functions_list + }) + + def save_state_machines(self, state_machines_list): + """ + Saves the state machines in the storage + + :param state_machines_list: Dict[str, str] of the state machines with key being the state + machine name and the value is the state machine code + """ + + assert isinstance(state_machines_list, dict) + + return self.__save_experiment_data('state-machine', { + 'experimentId': self.__experiment_id, + 'state_machines': state_machines_list + }) + + def save_brain(self, populations, pynn_script): + """ + Saves the populations and pynn_script + + :param populations: List of populations. + A population can be a list, with the format: {name:str, list:[int]} (list has neurons ids) + A population can be a slice, with the format: {name:str, from: int, to: int, step: int} + :param pynn_script: string with the PyNN script + """ + + assert isinstance(populations, list) + assert isinstance(pynn_script, str) + + return self.__save_experiment_data('brain', { + 'data': pynn_script, + 'additional_populations': populations + }) + + def save_csv(self): + """ + Saves the recorded csv data + """ + + if not self.__sim_url: + raise Exception("No simulation data!") + + url = '%s/%s' % (self.__sim_url, self.__config['simulation-services']['csv-recorders']) + + try: + res = requests.put(url, headers=self.__storage_headers, json={}) + + if res.status_code != httplib.OK: + raise Exception('Error status code %s' % res.status_code) + self.__logger.info("Saved CSV data") + except Exception as err: + self.__logger.info(err) + raise Exception("Failed to save CSV.") + + def save_world(self): + """ + Saves the world + """ + + return self.__save_experiment_data('sdf-world', {}, method='post') + def get_state_machine(self, state_machine_name): """ Gets the State Machine body for a given state machine name. diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py index 7940166..c5ebb2d 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py @@ -582,6 +582,107 @@ class TestSimulation(unittest.TestCase): None) self.assertRaises(Exception, sim.delete_state_machine, 'foo') + @patch('requests.post') + @patch('requests.put') + def test_save_experiment_data(self, mock_put, mock_post): + + sim = Simulation(BBPOIDCClient(), Config('local'), {}) + + exp_id = 'exp_id' + serverurl = 'serverurl' + sim._Simulation__experiment_id = exp_id + sim._Simulation__server_info = { + 'gzweb':{ + 'nrp-services': serverurl + } + } + + sim._Simulation__headers = {} + + class Response(object): + def __init__(self, status=200): + self.status_code = status + + res_OK = Response(200) + res_NOK = Response(500) + mock_put.return_value = res_NOK + + tfs = ["""@nrp.Robot2Neuron() +def benchmark_evaluation(t): + pass +""", +"""@nrp.Robot2Neuron() +def benchmark_evaluation2(t): + pass +"""] + + self.assertRaises(Exception, sim.save_transfer_functions, tfs) + + mock_put.return_value = res_OK + mock_put.reset_mock() + + sim.save_transfer_functions(tfs) + mock_put.assert_called_once_with('%s/experiment/%s/transfer-functions' % (serverurl, exp_id), + headers=sim._Simulation__headers, + json={'transfer_functions': tfs, 'experimentId': exp_id}) + + mock_put.reset_mock() + + state_machines = { + 'state_machine_0': '# bla bla 0', + 'state_machine_1': '# bla bla 1' + } + + sim.save_state_machines(state_machines) + + mock_put.assert_called_once_with('%s/experiment/%s/state-machines' % (serverurl, exp_id), + headers=sim._Simulation__headers, + json={'state_machines': state_machines, + 'experimentId': exp_id}) + + mock_put.reset_mock() + + populations = [ + { + 'name': 'pop1', + 'from': 0, + 'to': 1, + 'step': 1, + 'regex':"^\b(?!\bpop0\b)([A-z_]+[\w_]*)$" + } + ] + + brain = 'some brain code' + sim.save_brain(populations, brain) + + mock_put.assert_called_once_with('%s/experiment/%s/brain' % (serverurl, exp_id), + headers=sim._Simulation__headers, + json={'additional_populations': populations, + 'data': brain}) + + mock_post.return_value = res_OK + + sim.save_world() + mock_post.assert_called_once_with('%s/experiment/%s/sdf_world' % (serverurl, exp_id), + headers=sim._Simulation__headers, + json={}) + + mock_put.reset_mock() + mock_put.return_value = res_NOK + self.assertRaises(Exception, sim.save_csv) + + mock_put.reset_mock() + sim._Simulation__sim_url = 'url' + self.assertRaises(Exception, sim.save_csv) + + mock_put.reset_mock() + mock_put.return_value = res_OK + sim.save_csv() + + mock_put.assert_called_once_with('%s/csv-recorders' % sim._Simulation__sim_url, + headers=sim._Simulation__headers, + json={}) + @patch('sys.stdout', new_callable=StringIO) def test_print_scripts(self, mock_stdout): diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py index ec83b26..364f3cf 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py @@ -477,15 +477,29 @@ mock-server-5 @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') @patch('requests.post') - def test_clone_experiment_to_storage_fail(self, request, mock_list): - mock_list.return_value = ['missing_id'] + def test_clone_cloned_to_storage_fail(self, request, mock_list): + mock_list.return_value = ['exp_id'] class Request(object): status_code = 477 request.return_value = Request() + self._vc._VirtualCoach__storage_username = None + self.assertRaises(ValueError, self._vc.clone_cloned_experiment, 'exp_id') self._vc._VirtualCoach__storage_username = 'token' - self.assertRaises(Exception, self._vc.clone_cloned_experiment, 'missing_id') + self.assertRaises(Exception, self._vc.clone_cloned_experiment, 'exp_id') + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + @patch('requests.post') + def test_clone_cloned_to_storage(self, request, mock_list): + mock_list.return_value = ['exp_id'] + + class Request(object): + status_code = 200 + + request.return_value = Request() + self._vc._VirtualCoach__storage_username = 'token' + self._vc.clone_cloned_experiment('exp_id') @patch('getpass.getpass', return_value='password') -- GitLab