diff --git a/examples/integration_test/it.py b/examples/integration_test/it.py index 0ff25fdec37920be744a4527f07e423152897f82..04449d32ec1ddc6029d92a7e67ccf33201ce065b 100644 --- a/examples/integration_test/it.py +++ b/examples/integration_test/it.py @@ -281,7 +281,7 @@ def tf(t): 'simulation.') results.done(True) - results.start("Deleting a transfer functino") + results.start("Deleting a transfer function") sim.delete_transfer_function('tf') tfs = sim._Simulation__get_simulation_scripts('transfer-function')['data'] if 'tf' in tfs: diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py index b4cc360b60859d52affcc1610d92a8a160eafdda..27e454da8d17ca7dfc9ad7ea20c3c0d9ec8dd2cb 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py @@ -47,7 +47,7 @@ class Config(dict): Load and validate the configuration file. Update all proxy service parameters to use the given environment. - :param environment A string for the environment to be used (e.g. 'staging', 'dev', + :param environment A string for the environment to be used (e.g. 'production', 'dev', 'local', or a custom value defined in the config.json). If None is provided, then the environment will be parsed from the version of this package. @@ -83,7 +83,9 @@ class Config(dict): # validate required sections of the config, except if any values are missing self.__validate('oidc', ['user']) self.__validate('proxy', ['staging', 'dev', 'local', environment]) - self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info']) + self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info', + 'experiment-clone', 'storage-authentication', + 'storage-experiment-list']) self.__validate('simulation-services', ['create', 'state', 'reset', 'csv-recorders']) self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain']) self.__validate('reset-services', ['robot_pose', 'full', 'world', 'brain']) diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py index c988b4fff63be643043a8bd3ed2faeb0e246c6d0..66cd0b3ac7118a3aee471721c6324472117ba5cc 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py @@ -30,6 +30,7 @@ from hbp_nrp_virtual_coach.config import Config from bbp_client.oidc.client import BBPOIDCClient import httplib +import requests import json import logging from urllib2 import HTTPError @@ -68,7 +69,8 @@ class Simulation(object): self.__logger = logging.getLogger('Simulation') # pylint: disable=too-many-locals - def launch(self, experiment_id, experiment_conf, server, reservation): + def launch(self, experiment_id, experiment_conf, server, reservation, cloned=False, + headers=None): """ Attempt to launch and initialize the given experiment on the given servers. This should not be directly invoked by users, use the VirtualCoach interface to validate @@ -79,11 +81,17 @@ class Simulation(object): :param experiment_conf: A string representing the configuration file for the experiment. :param server: A string representing the name of the server to try to launch on. :param reservation: A string representing a cluster resource reservation (if any). + :param cloned: (optional) A flag indicating whether the user wants to launch a cloned + experiment or not. + :param headers: (optional) Request header including Storage Server token, in case we want + to launch a cloned experiment. """ assert isinstance(experiment_id, str) assert isinstance(experiment_conf, str) assert isinstance(server, str) assert isinstance(reservation, (str, type(None))) + assert isinstance(cloned, bool) + assert isinstance(headers, (dict, type(None))) # do not allow reuse of this instance if a simulation has been launched if self.__server: @@ -103,21 +111,30 @@ class Simulation(object): self.__server_info = json.loads(server_json) # attempt to launch the simulation with given parameters on the server + url = '%s/%s' % (self.__server_info['gzweb']['nrp-services'], + self.__config['simulation-services']['create']) sim_info = {'brainProcesses': 1, 'experimentConfiguration': experiment_conf, 'gzserverHost': self.__server_info['serverJobLocation'], 'reservation': reservation} - url = '%s/%s' % (self.__server_info['gzweb']['nrp-services'], - self.__config['simulation-services']['create']) - res, sim_json = self.__oidc_client.request(url, method='POST', - body=json.dumps(sim_info)) - + if cloned: + # append extra information to the sim_info dictionary that are required for + # launching cloned simulations + sim_info['experimentID'] = experiment_id + sim_info['private'] = True + res = requests.post(url, headers=headers, json=sim_info) + sim_json = res.content + status = res.status_code + else: + res, sim_json = self.__oidc_client.request(url, method='POST', + body=json.dumps(sim_info)) + status = int(res['status']) # check to see if the launch was successful, any other failure return codes # such as 404 will trigger an exception by the OIDCClient itself - if res['status'] == str(httplib.CONFLICT): + if status == httplib.CONFLICT: raise Exception('Simulation server is launching another experiment.') - elif res['status'] != str(httplib.CREATED): + elif status != httplib.CREATED: raise Exception("Simulation responded with HTTP status %s" % str(res['status'])) # retrieve and store the simulation information @@ -509,7 +526,7 @@ class Simulation(object): """ Adds a new transfer function to the simulation. - :param transfer_function A string containing the new transfer function code. + :param transfer_function: A string containing the new transfer function code. """ self.__set_script('transfer-function', transfer_function, new=True) @@ -517,7 +534,7 @@ class Simulation(object): """ Deletes a transfer function. - :param transfer_function_name A string containing the name of the transfer function to be + :param transfer_function_name: A string containing the name of the transfer function to be deleted. """ self.__delete_script('transfer-function', transfer_function_name) diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py index 3c367accd7bd0d2a4846fa592370e18cb41387e0..5c964663ccdd2ff7e603b273cf8d18d9b2944e49 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py @@ -127,6 +127,34 @@ class TestSimulation(unittest.TestCase): # calling launch twice on an instance should fail after successful creation self.assertRaises(Exception, sim.launch, 'id', 'conf', 'server-name', 'reservation') + @patch('requests.post') + def test_create_cloned(self, mock_request): + + # this will create a sim, don't store it in class since we can't guarantee order + sim = Simulation(BBPOIDCClient(), Config('local')) + + class Response(object): + status_code = 201 + content = '{"simulationID": "12"}' + + mock_request.return_value = Response() + + # mock OIDC calls to handle request type + def oidc_mock(url, method=None, body=None): + # server-info lookup + return None, '{"serverJobLocation": "mock-location",' \ + '"gzweb": {"nrp-services": "mock-services"}}' + + sim._Simulation__oidc_client.request = Mock() + sim._Simulation__oidc_client.request.side_effect = oidc_mock + + # mock the call to set simulation state + sim._Simulation__set_state = Mock() + + self.assertEqual(sim.launch('id', 'conf', 'server-name', 'reservation', cloned=True, + headers={}), True) + self.assertEqual(sim._Simulation__oidc_client.request.call_count, 1) + def test_create_with_rospy(self): # this will create a sim, don't store it in class since we can't guarantee order @@ -136,7 +164,8 @@ class TestSimulation(unittest.TestCase): def oidc_mock(url, method=None, body=None): # server-info lookup if method is None: - return None, '{"serverJobLocation": "mock-location", "gzweb": {"nrp-services": "mock-services"}}' + return None, '{"serverJobLocation": "mock-location",' \ + '"gzweb": {"nrp-services": "mock-services"}}' # POST for the creation request return {'status': str(httplib.CREATED)}, '{"simulationID": "12"}' @@ -159,8 +188,12 @@ class TestSimulation(unittest.TestCase): with patch('__builtin__.__import__', side_effect=mock_import): self.assertEqual(sim.launch('id', 'conf', 'server-name', 'reservation'), True) - mock_rospy.Subscriber.assert_has_calls([call('/ros_cle_simulation/status', mock_rospy.String,sim._Simulation__on_status), - call('/ros_cle_simulation/cle_error', mock_rospy.CLEError,sim._Simulation__on_error)]) + mock_rospy.Subscriber.assert_has_calls([call('/ros_cle_simulation/status', + mock_rospy.String, + sim._Simulation__on_status), + call('/ros_cle_simulation/cle_error', + mock_rospy.CLEError, + sim._Simulation__on_error)]) def test_create_without_rospy(self): @@ -171,7 +204,8 @@ class TestSimulation(unittest.TestCase): def oidc_mock(url, method=None, body=None): # server-info lookup if method is None: - return None, '{"serverJobLocation": "mock-location", "gzweb": {"nrp-services": "mock-services"}}' + return None, '{"serverJobLocation": "mock-location",' \ + '"gzweb": {"nrp-services": "mock-services"}}' # POST for the creation request return {'status': str(httplib.CREATED)}, '{"simulationID": "12"}' @@ -568,17 +602,20 @@ class TestSimulation(unittest.TestCase): # ignored duplicate registration sim.register_status_callback(mock_callback) self.assertEqual(sim._Simulation__status_callbacks, [mock_callback]) - sim._Simulation__logger.warn.assert_called_once_with('Attempting to register duplicate status callback, ignoring.') + sim._Simulation__logger.warn.assert_called_once_with('Attempting to register duplicate ' + 'status callback, ignoring.') self.assertEqual(sim._Simulation__logger.info.call_count, 1) def test_on_error(self): sim = Simulation(BBPOIDCClient(), Config('local')) sim._Simulation__logger = Mock() + class MockMsg(object): errorType = "test" sourceType = "test script" + def __str__(self): - return "test body message" + return "test body message" # progress message that should be logged error_msg = "There was a test error resulting from the test script." \ @@ -586,7 +623,6 @@ class TestSimulation(unittest.TestCase): sim._Simulation__on_error(MockMsg()) sim._Simulation__logger.error.assert_called_once_with(error_msg) - def test_on_status(self): # this will create a sim, don't store it in class since we can't guarantee order @@ -595,6 +631,7 @@ class TestSimulation(unittest.TestCase): sim._Simulation__logger = Mock() sim._Simulation__status_sub = Mock() sim._Simulation__error_sub = Mock() + sim._Simulation__previous_subtask = 'foo' # callback to ensure it is only called when appropriate mock_callback = Mock() @@ -605,7 +642,9 @@ class TestSimulation(unittest.TestCase): self.data = json.dumps(msg) # progress message that should be discarded + sim._Simulation__on_status(MockString('action')) sim._Simulation__on_status(MockString({'progress': {'done': True}})) + sim._Simulation__on_status(MockString({'progress': {'subtask': 'foo'}})) self.assertEqual(sim._Simulation__logger.info.call_count, 0) self.assertEqual(sim._Simulation__status_sub.unregister.call_count, 0) self.assertEqual(mock_callback.call_count, 0) diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py index 19ef25429674f7d787679123df5ec9c7212793c2..1eb3d52dfaadb8120f5602b06b665bf5192a022e 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py @@ -31,6 +31,8 @@ from bbp_client.oidc.client import BBPOIDCClient from mock import Mock, patch import unittest +import requests +import logging import copy from dateutil import parser @@ -46,6 +48,7 @@ class TestVirtualCoach(unittest.TestCase): # overriden with Mock for specific test case behavior as needed (see below) import __builtin__ as builtins realimport = builtins.__import__ + def rospy_import_fail(name, globals=globals(), locals=locals(), fromlist=[], level=-1): if name == 'rospy': raise ImportError('no ROS for tests') @@ -54,10 +57,10 @@ class TestVirtualCoach(unittest.TestCase): self._vc = VirtualCoach() - self._mock_available_servers_list = [ - {"id": 'mock-server-1'}, - {"id":'mock-server-4'}, - {"id":'mock-server-5'} + self._mock_available_servers_list = [ + {"id": 'mock-server-1'}, + {"id": 'mock-server-4'}, + {"id": 'mock-server-5'} ] self._mock_exp_list = {'MockExperiment1': {'configuration': {'name': 'C', @@ -73,14 +76,14 @@ class TestVirtualCoach(unittest.TestCase): 'runningSimulation': {'owner': '3', 'state': 'started', 'creationDate': 'Feb 03 12:15:24 UTC 2017'}}] - }, + }, 'MockExperiment2': {'configuration': {'name': 'A', 'maturity': 'production', 'description': 'Mock A', 'timeout': 900, 'experimentConfiguration': 'foo/bar3.xml'}, 'joinableServers': [] - }, + }, 'MockDevExperiment': {'configuration': {'name': 'B', 'maturity': 'development', 'description': 'Mock B', @@ -90,7 +93,7 @@ class TestVirtualCoach(unittest.TestCase): 'runningSimulation': {'owner': '2', 'state': 'created', 'creationDate': 'Feb 03 12:07:58 UTC 2017'}}] - }} + }} self._mock_exp_list_local = {'MockExperiment1': {'configuration': {'name': 'C', 'maturity': 'production', @@ -101,7 +104,7 @@ class TestVirtualCoach(unittest.TestCase): 'runningSimulation': {'owner': '1', 'state': 'paused', 'creationDate': 'Feb 03 12:11:10 UTC 2017'}}] - }, + }, 'MockExperiment2': {'configuration': {'name': 'A', 'maturity': 'production', 'description': 'Mock A', @@ -110,21 +113,33 @@ class TestVirtualCoach(unittest.TestCase): 'joinableServers': [] }} + self._mock_exp_list_cloned = [{'uuid': 'MockExperiment1_0', 'name': 'MockExperiment1_0'}, + {'uuid': 'MockExperiment2_0', 'name': 'MockExperiment2_0'}] + def test_init_asserts(self): - # invalid username + # invalid oidc username self.assertRaises(AssertionError, VirtualCoach, oidc_username=123) # invalid environment self.assertRaises(AssertionError, VirtualCoach, environment=True) + # invalid storage server username + self.assertRaises(AssertionError, VirtualCoach, storage_username=123) + @patch('bbp_client.oidc.client.BBPOIDCClient.implicit_auth') - def test_init_login(self, mock_login): + def test_init_oidc_login(self, mock_login): # mocked OIDC authentication, ensure called if username provided mock_login.returnValue = BBPOIDCClient() oidc_vc = VirtualCoach(oidc_username='foo') mock_login.assert_called_once_with('foo') + @patch('getpass.getpass', return_value='password') + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_storage_token') + def test_init_storage_login(self, mock_login, mock_getpass): + storage_vc = VirtualCoach(storage_username='user') + mock_login.assert_called_once_with('user', 'password') + def test_init_rospy(self): # mock the rospy import since we can't include it as a dependency for the package yet @@ -152,13 +167,13 @@ class TestVirtualCoach(unittest.TestCase): self._vc.print_experiment_list() prod_table = """ -+-----------------+------+-------------+ -| Configuration | Name | Description | -+=================+======+=============+ -| MockExperiment2 | A | Mock A | -+-----------------+------+-------------+ -| MockExperiment1 | C | Mock C | -+-----------------+------+-------------+ ++-----------------+------+--------------------+-------------+ +| Configuration | Name | Configuration Path | Description | ++=================+======+====================+=============+ +| MockExperiment2 | A | foo/bar3.xml | Mock A | ++-----------------+------+--------------------+-------------+ +| MockExperiment1 | C | foo/bar1.xml | Mock C | ++-----------------+------+--------------------+-------------+ """ self.assertEqual(mock_stdout.getvalue().strip(), prod_table.strip()) @@ -171,15 +186,15 @@ class TestVirtualCoach(unittest.TestCase): self._vc.print_experiment_list(True) dev_table = """ -+---------------------------------+------+-------------+ -| Configuration | Name | Description | -+=================================+======+=============+ -| MockExperiment2 (production) | A | Mock A | -+---------------------------------+------+-------------+ -| MockDevExperiment (development) | B | Mock B | -+---------------------------------+------+-------------+ -| MockExperiment1 (production) | C | Mock C | -+---------------------------------+------+-------------+ ++---------------------------------+------+--------------------+-------------+ +| Configuration | Name | Configuration Path | Description | ++=================================+======+====================+=============+ +| MockExperiment2 (production) | A | foo/bar3.xml | Mock A | ++---------------------------------+------+--------------------+-------------+ +| MockDevExperiment (development) | B | foo/bar2.xml | Mock B | ++---------------------------------+------+--------------------+-------------+ +| MockExperiment1 (production) | C | foo/bar1.xml | Mock C | ++---------------------------------+------+--------------------+-------------+ """ self.assertEqual(mock_stdout.getvalue().strip(), dev_table.strip()) @@ -205,50 +220,51 @@ class TestVirtualCoach(unittest.TestCase): """ self.assertEqual(mock_stdout.getvalue().strip(), running_table.strip()) - @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') - @patch('hbp_nrp_virtual_coach.virtual_coach.datetime') - @patch('sys.stdout', new_callable=StringIO) - def test_print_running_experiments(self, mock_stdout, mock_date, mock_list): - - # mock the OIDC server call - mock_list.return_value = self._mock_exp_list - - # mock the user lookup OIDC call - def fake_user_lookup(url): - i = url.rsplit('/')[-1] - if i == '1': - return (None, json.dumps({'displayName': 'User One'})) - if i == '2': - return (None, json.dumps({'displayName': 'User Two'})) - if i == '3': - return (None, json.dumps({'displayName': 'User Three'})) - - self._vc._VirtualCoach__oidc_client.request = Mock() - self._vc._VirtualCoach__oidc_client.request.side_effect = fake_user_lookup - - # mock the current time (so that the table prints consistently) - mock_date.now = Mock(return_value=parser.parse('Feb 03 12:20:03 UTC 2017')) - - self._vc.print_running_experiments() - - running_table = """ - +-------------------+------------+--------+---------+---------+---------------+ - | Configuration | Owner | Time | Timeout | State | Server | - +===================+============+========+=========+=========+===============+ - | MockDevExperiment | User Two | 0:12:0 | 0:13:00 | created | mock-server-2 | - +-------------------+------------+--------+---------+---------+---------------+ - | MockExperiment1 | User One | 0:08:5 | 0:14:00 | paused | mock-server-3 | - +-------------------+------------+--------+---------+---------+---------------+ - | MockExperiment1 | User Three | 0:04:3 | 0:14:00 | started | mock-server-6 | - +-------------------+------------+--------+---------+---------+---------------+ - """ - self.assertEqual(mock_stdout.getvalue().strip(), running_table.strip()) + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + @patch('hbp_nrp_virtual_coach.virtual_coach.datetime') + @patch('sys.stdout', new_callable=StringIO) + def test_print_running_experiments(self, mock_stdout, mock_date, mock_list): + + self._vc._VirtualCoach__oidc_username = 'user' + # mock the OIDC server call + mock_list.return_value = self._mock_exp_list + + # mock the user lookup OIDC call + def fake_user_lookup(url): + i = url.rsplit('/')[-1] + if i == '1': + return None, json.dumps({'displayName': 'User One'}) + if i == '2': + return None, json.dumps({'displayName': 'User Two'}) + if i == '3': + return None, json.dumps({'displayName': 'User Three'}) + + self._vc._VirtualCoach__oidc_client.request = Mock() + self._vc._VirtualCoach__oidc_client.request.side_effect = fake_user_lookup + + # mock the current time (so that the table prints consistently) + mock_date.now = Mock(return_value=parser.parse('Feb 03 12:20:03 UTC 2017')) + + self._vc.print_running_experiments() + + running_table = """ ++-------------------+------------+--------+---------+---------+---------------+ +| Configuration | Owner | Time | Timeout | State | Server | ++===================+============+========+=========+=========+===============+ +| MockDevExperiment | User Two | 0:12:0 | 0:13:00 | created | mock-server-2 | ++-------------------+------------+--------+---------+---------+---------------+ +| MockExperiment1 | User One | 0:08:5 | 0:14:00 | paused | mock-server-3 | ++-------------------+------------+--------+---------+---------+---------------+ +| MockExperiment1 | User Three | 0:04:3 | 0:14:00 | started | mock-server-6 | ++-------------------+------------+--------+---------+---------+---------------+ + """ + self.assertEqual(mock_stdout.getvalue().strip(), running_table.strip()) @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list') @patch('sys.stdout', new_callable=StringIO) def test_print_available_servers(self, mock_stdout, available_servers): - # mock the OIDC server call + # mock the GET server call available_servers.return_value = self._mock_available_servers_list self._vc.print_available_servers() @@ -264,7 +280,7 @@ mock-server-5 def test_print_no_available_servers(self, mock_stdout, available_servers): # mock the OIDC server call - available_servers.return_value = [] + available_servers.return_value = [] self._vc.print_available_servers() available_servers = 'No available servers.' @@ -272,12 +288,21 @@ mock-server-5 def test_get_experiment_list(self): - # mock the OIDC request - self._vc._VirtualCoach__oidc_client.request = Mock(return_value=(None, json.dumps(self._mock_exp_list))) + # mock the request + self._vc._VirtualCoach__oidc_client.request = Mock() + self._vc._VirtualCoach__oidc_client.request.return_value = 'x', json.dumps(self._mock_exp_list) list_json = self._vc._VirtualCoach__get_experiment_list() self.assertEqual(list_json, self._mock_exp_list) + @patch('requests.get') + def test_get_cloned_experiment_list(self, mock_request): + mock_response = requests.Response + mock_response.content = json.dumps(self._mock_exp_list_cloned) + mock_request.return_value = mock_response + exp_list = self._vc._VirtualCoach__get_experiment_list(cloned=True) + self.assertEqual(exp_list, ['MockExperiment1_0', 'MockExperiment2_0']) + def test_launch_asserts(self): self.assertRaises(AssertionError, self._vc.launch_experiment, None) self.assertRaises(AssertionError, self._vc.launch_experiment, 'foo', True) @@ -291,10 +316,20 @@ mock-server-5 self.assertRaises(ValueError, self._vc.launch_experiment, 'InvalidExperimentID') @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') - def test_launch_invalid_server(self, mock_list): + def test_launch_cloned_experiment_without_storage(self, mock_list): + mock_list.return_value = self._mock_exp_list + self._vc._VirtualCoach__token = None + self.assertRaises(ValueError, self._vc.launch_experiment, 'MockExperiment1', cloned=True) - # mock the OIDC server call + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list') + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + def test_launch_invalid_server(self, mock_list, servers_list): + + # mock the experiments and servers call mock_list.return_value = self._mock_exp_list + servers_list.return_value = self._mock_available_servers_list + self.assertRaises(ValueError, self._vc.launch_experiment, 'MockExperiment1', 'invalid-server-1') @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list') @@ -316,27 +351,32 @@ mock-server-5 mock_sim.return_value = None mock_sim_launch.return_value = True + # mock the GET server call servers_list.return_value = self._mock_available_servers_list - # mock the OIDC server call + mock_list.return_value = self._mock_exp_list self._vc.launch_experiment('MockExperiment1', 'mock-server-4') + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list') @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') @patch('hbp_nrp_virtual_coach.simulation.Simulation.__init__') @patch('hbp_nrp_virtual_coach.simulation.Simulation.launch') - def test_launch_any(self, mock_sim_launch, mock_sim, mock_list): + def test_launch_any(self, mock_sim_launch, mock_sim, mock_list, servers_list): # mock sim launch to succeed mock_sim.return_value = None - def launch(experiment_id, experiment_conf, server, reservation): + self._vc._VirtualCoach__storage_username = 'username' + + def launch(experiment_id, experiment_conf, server, reservation, cloned, headers): if server == 'mock-server-1': raise Exception('fake failure!') return True mock_sim_launch.side_effect = launch - # mock the OIDC server call + # mock the experiments and servers call mock_list.return_value = self._mock_exp_list - self._vc.launch_experiment('MockExperiment1') + servers_list.return_value = self._mock_available_servers_list + self._vc.launch_experiment('MockExperiment1', cloned=True) @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') @patch('hbp_nrp_virtual_coach.simulation.Simulation.__init__') @@ -350,3 +390,89 @@ mock-server-5 # mock the OIDC server call mock_list.return_value = self._mock_exp_list self.assertRaises(Exception, self._vc.launch_experiment, 'MockExperiment1') + + def test_clone_experiment_to_storage_assert(self): + self.assertRaises(AssertionError, self._vc.clone_experiment_to_storage, 123) + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + def test_clone_experiment_without_storage(self, mock_list): + mock_list.return_value = self._mock_exp_list + self.assertRaises(ValueError, self._vc.clone_experiment_to_storage, 'foo/bar1.xml') + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + @patch('requests.post') + def test_clone_experiment_to_storage_fail(self, request, mock_list): + mock_list.return_value = self._mock_exp_list + + class Request(object): + status_code = 477 + + request.return_value = Request() + self._vc._VirtualCoach__storage_username = 'token' + self.assertRaises(Exception, self._vc.clone_experiment_to_storage, 'foo/bar1.xml') + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + def test_clone_invalid_experiment(self, mock_list): + mock_list.return_value = self._mock_exp_list + self.assertRaises(ValueError, self._vc.clone_experiment_to_storage, 'invalid_configuration') + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + @patch('hbp_nrp_virtual_coach.virtual_coach.logger.info') + @patch('requests.post') + def test_clone_experiment_to_storage(self, request, mock_logger, mock_list): + mock_list.return_value = self._mock_exp_list + + class Request(object): + status_code = 200 + + request.return_value = Request() + self._vc._VirtualCoach__storage_username = 'username' + self._vc.clone_experiment_to_storage('foo/bar1.xml') + mock_logger.assert_called_once() + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + def test_print_cloned_experiments_fail(self, mock_list): + mock_list.return_value = self._mock_exp_list + self._vc._VirtualCoach__token = None + self.assertRaises(ValueError, self._vc.print_cloned_experiments) + + @patch('hbp_nrp_virtual_coach.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list') + @patch('sys.stdout', new_callable=StringIO) + def test_print_cloned_experiments(self, mock_stdout, mock_list): + self._vc._VirtualCoach__storage_username = 'token' + mock_list.return_value = ['MockExperiment1_0', 'MockExperiment2_0'] + self._vc.print_cloned_experiments() + cloned_experiments = """ ++-------------------+ +| Name | ++===================+ +| MockExperiment1_0 | ++-------------------+ +| MockExperiment2_0 | ++-------------------+ + """ + self.assertEqual(mock_stdout.getvalue().strip(), cloned_experiments.strip()) + + def test_get_storage_token_asserts(self): + self.assertRaises(AssertionError, self._vc._VirtualCoach__get_storage_token, 123, 'foo') + self.assertRaises(AssertionError, self._vc._VirtualCoach__get_storage_token, 'foo', 123) + + @patch('requests.post') + def test_get_storage_token_fail(self, mock_request): + + class Response(object): + status_code = 500 + + mock_request.return_value = Response() + self.assertRaises(Exception, self._vc._VirtualCoach__get_storage_token, 'user', 'pass') + + @patch('requests.post') + def test_get_storage_token(self, mock_request): + + class Response(object): + status_code = 200 + content = 'token' + + mock_request.return_value = Response() + content = self._vc._VirtualCoach__get_storage_token('user', 'pass') + self.assertEqual(content, 'token') diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py index d312c7a02a3808c92220148e90a96e8fe1671a1a..b6b0eb96cf517db5acb5082d6fd8e2b0e08fb0c9 100644 --- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py +++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py @@ -33,7 +33,9 @@ from bbp_client.oidc.client import BBPOIDCClient from datetime import datetime, timedelta from dateutil import parser, tz import json +import requests import logging +import getpass from texttable import Texttable logger_format = '%(levelname)s: [%(asctime)s - %(name)s] %(message)s' @@ -48,7 +50,7 @@ class VirtualCoach(object): view available experiments, query currently running experiments, launch a simulation, and more. """ - def __init__(self, environment=None, oidc_username=None): + def __init__(self, environment=None, oidc_username=None, storage_username=None): """ Instantiates the Virtual Coach by loading the configuration file and logging into OIDC for the given user. This will only fail if the config file is invalid or if the user @@ -63,9 +65,13 @@ class VirtualCoach(object): user, required if the provided environment requires OIDC authentication. The user will be interactively asked for a password by the OIDC client if the token is expired or they have not logged in. + :param storage_username: (optional) A string representing the Storage Server username. It is + required if the user wants to have access to the storage server to + clone experiments and launch cloned experiments. """ assert isinstance(environment, (str, type(None))) assert isinstance(oidc_username, (str, type(None))) + assert isinstance(storage_username, (str, type(None))) # ROS node and logger configuration only if rospy is available # pylint: disable=import-error @@ -92,15 +98,30 @@ class VirtualCoach(object): self.__config = Config(environment) self.__oidc_username = oidc_username + self.__storage_username = storage_username # if an OIDC username is provided, attempt to login or retrieve the last valid token if oidc_username: # this will interactively prompt the user for a password in terminal if needed logger.info('Logging into OIDC as: %s', oidc_username) self.__oidc_client = BBPOIDCClient.implicit_auth(oidc_username) + # if a Storage Server username is provided, attempt to login + elif storage_username: + # this will interactively prompt the user for a password in terminal + logger.warn('No OIDC username supplied, simulation services will fail if OIDC is ' + 'enabled in this environment (%s).', environment) + logger.info('Logging into the Storage Server as: %s', storage_username) + storage_password = getpass.getpass() + self.__storage_headers = {'Content-Type': 'application/json', + 'Authorization': 'Bearer %s' % self.__get_storage_token( + storage_username, storage_password)} + self.__oidc_client = BBPOIDCClient() else: # use an unauthenticated client for local installs to reuse the same API logger.warn('No OIDC username supplied, simulation services will fail if OIDC is ' 'enabled in this environment (%s).', environment) + logger.warn('No Storage Server username supplied, access to the Storage Server will be ' + 'denied.') + self.__storage_headers = {} self.__oidc_client = BBPOIDCClient() # if the config is valid and the login doesn't fail, we're ready @@ -121,14 +142,15 @@ class VirtualCoach(object): # construct the table of experiments with only minimal useful information table = Texttable() - table.header(['Configuration', 'Name', 'Description']) + table.header(['Configuration', 'Name', 'Configuration Path', 'Description']) for name, v in sorted(exp_list.iteritems(), key=lambda x: x[1]['configuration']['name']): if v['configuration']['maturity'] != 'production' and not dev: continue if dev: name = '%s (%s)' % (name, v['configuration']['maturity']) desc = ' '.join(v['configuration']['description'].strip().split()) - table.add_row([name, v['configuration']['name'], desc]) + table.add_row([name, v['configuration']['name'], + v['configuration']['experimentConfiguration'], desc]) # display the table logger.info('List of production%s experiments:', '' if not dev else ' and development') @@ -195,7 +217,7 @@ class VirtualCoach(object): logger.info('Available servers:') print '\n'.join(servers) - def launch_experiment(self, experiment_id, server=None, reservation=None): + def launch_experiment(self, experiment_id, server=None, reservation=None, cloned=False): """ Attempts to launch a simulation with the given parameters. If no server is explicitly given then all available backend servers will be tried. @@ -206,25 +228,36 @@ class VirtualCoach(object): provided, then all backend servers will be checked. :param reservation: (optional) A cluster reservation string if the user has reserved specific resources, otherwise use any available resources. + :param cloned: (optional) Flag to launch a cloned experiment from the Storage Server. """ assert isinstance(experiment_id, str) assert isinstance(server, (str, type(None))) assert isinstance(reservation, (str, type(None))) + assert isinstance(cloned, bool) # retrieve the experiment list to verify that the given id is valid for the backend logger.info('Preparing to launch %s.', experiment_id) - exp_list = self.__get_experiment_list() + exp_list = self.__get_experiment_list(cloned=cloned) if experiment_id not in exp_list: - raise ValueError('Experiment ID: "%s" is invalid, please check the experiment list.' % - experiment_id) + raise ValueError('Experiment ID: "%s" is invalid, please check the experiment list. %s' + % (experiment_id, exp_list)) + + if cloned and self.__storage_username is None: + raise ValueError('The cloned experiment %s cannot be launched. No Storage Server ' + 'credentials found. To be able to launch cloned experiments, you have ' + 'to instantiate the Virtual Coach either with the storage_username ' + 'parameter and login successfully' % experiment_id) # get the experiment configuration details and available servers that can be used available_servers = self.__get_available_server_list() from pprint import pprint pprint(available_servers) servers = [available_server['id'] for available_server in available_servers] - experiment = exp_list[experiment_id] - experiment_conf = experiment['configuration']['experimentConfiguration'] + if cloned: + experiment_conf = "" + else: + experiment = exp_list[experiment_id] + experiment_conf = experiment['configuration']['experimentConfiguration'] # if the user provided a specific server, ensure it is available before trying to launch if server is not None: @@ -242,7 +275,8 @@ class VirtualCoach(object): sim = Simulation(self.__oidc_client, self.__config) for server in servers: try: - if sim.launch(experiment_id, str(experiment_conf), str(server), reservation): + if sim.launch(experiment_id, str(experiment_conf), str(server), reservation, + cloned=cloned, headers=self.__storage_headers): return sim # pylint: disable=broad-except @@ -253,20 +287,100 @@ class VirtualCoach(object): # simulation launch unsuccessful, abort raise Exception('Simulation launch failed, consult the logs or try again later.') - def __get_experiment_list(self): + def clone_experiment_to_storage(self, exp_configuration_path): + """ + Attempts the clone an experiment to the Storage Server. Only works if the Virtual Coach was + instantiated with Storage Server support, i.e. Storage Server credentials + + :param exp_configuration_path: The path to the Experiment Configuration File. E.g., + braitenberg_husky/ExDXMLExample.exc when cloning the braitenberg husky experiment + """ + assert isinstance(exp_configuration_path, str) + + exp = self.__get_experiment_list() + config_paths = [exp[e]['configuration']['experimentConfiguration'] for e in exp] + if exp_configuration_path not in config_paths: + raise ValueError('Experiment Configuration Path: %s is invalid, please check this list ' + 'of all Configuration Paths:\n%s' % (exp_configuration_path, + '\n'.join(config_paths))) + exp_name = exp_configuration_path.split('/')[0] + # Raise Error in case no storage server token available. To get the token, the VC has to be + # instantiated with the storage_username parameter + if self.__storage_username is None: + raise ValueError('The Experiment %s cannot be cloned! No Storage Server credentials ' + 'found. To be able to clone experiments, you have to instantiate the ' + 'Virtual Coach either with the storage_username parameter or the ' + 'oidc_username parameter and login successfully' % exp_name) + + body = {'expPath': exp_configuration_path} + res = requests.post(self.__config['proxy-services']['experiment-clone'], json=body, + headers=self.__storage_headers) + if res.status_code != 200: + raise Exception('Cloning Experiment failed, Status Code: %s' % res.status_code) + else: + logger.info('Experiment "%s" cloned successfully', exp_name) + + def print_cloned_experiments(self): + """ + Prints the list of the cloned experiments' names. Only works if the Virtual Coach was + instantiated with Storage Server support, i.e. with Storage Server credentials + """ + if self.__storage_username is None: + raise ValueError('Cloned experiments cannot be displayed! No Storage Server credentials' + ' found. To have access to the Storage Server, you have to instantiate' + ' the Virtual Coach either with the storage_username parameter or the ' + 'oidc_username parameter and login successfully') + exp_list = self.__get_experiment_list(cloned=True) + table = Texttable() + table.header(['Name']) + for experiment in exp_list: + table.add_row([experiment]) + print table.draw() + + def __get_storage_token(self, user_name, password): + """ + Attempts to acquire a storage server token based on the provided credentials + + :param user_name: string representing the Storage Server username + :param password: string representing the Storage Server password + """ + assert isinstance(user_name, str) + assert isinstance(password, str) + + response = requests.post(self.__config['proxy-services']['storage-authentication'], + json={'user': user_name, 'password': password}) + if response.status_code != 200: + raise Exception('Storage Server authentication failed, Status Code: %d' + % response.status_code) + else: + return response.content + + def __get_experiment_list(self, cloned=False): """ Internal helper to retrieve and parse the experiment list from the backend proxy. + + :param cloned: (optional) Flag to get cloned experiments to the storage """ + assert isinstance(cloned, bool) logger.info('Retrieving list of experiments.') - _, l_json = self.__oidc_client.request(self.__config['proxy-services']['experiment-list']) - return json.loads(l_json) + if cloned: + url = self.__config['proxy-services']['storage-experiment-list'] + response = requests.get(url, headers=self.__storage_headers) + # return a simple list containing only experiment names since this is the only + # information in the dictionary anyway + return [experiment['name'] for experiment in json.loads(response.content)] + else: + _, response = self.__oidc_client.request( + self.__config['proxy-services']['experiment-list']) + return json.loads(response) def __get_available_server_list(self): """ Internal helper to retrieve the available server list from the backend proxy. """ - logger.info('Retrieving list of available servers.') - _, l_json = self.__oidc_client.request(self.__config['proxy-services']['available-servers']) - return json.loads(l_json) + _, response = self.__oidc_client.request( + self.__config['proxy-services']['available-servers']) + + return json.loads(response)