From 97ef6b26e34d7ada73a957f75fadd74db198df4a Mon Sep 17 00:00:00 2001
From: Benedikt Feldotto <benedikt.feldotto@tum.de>
Date: Wed, 15 Dec 2021 10:15:22 +0000
Subject: [PATCH] Merged in NRRPLT-8412-virtual-coach-models (pull request #37)
NRRPLT-8412-virtual-coach-models
* [NRRPLT-8412] Print, delete and import models Virtual Coach functions + tests
* [NRRPLT-8412] Fix pylint errors
* [NRRPLT-8412] remove redundant keyword arg
* [NRRPLT-8412] fix error unclosed file
* [NRRPLT-8412] fix error unclosed file in import model
* [NRRPLT-8412] fix type variable and condition
* [NRRPLT-8412] Fix failing tests and pycodestyle
Change-Id: Id7b95518ed4270468c798a76ae47e8cf740626e2
Approved-by: Eloy Retamino
Approved-by: Vahid Zolfaghari
---
hbp_nrp_virtual_coach/pynrp/config.json | 3 +
hbp_nrp_virtual_coach/pynrp/config.py | 4 +-
.../pynrp/tests/test_virtual_coach.py | 63 +++++++++-
hbp_nrp_virtual_coach/pynrp/virtual_coach.py | 110 +++++++++++++++++-
4 files changed, 174 insertions(+), 6 deletions(-)
diff --git a/hbp_nrp_virtual_coach/pynrp/config.json b/hbp_nrp_virtual_coach/pynrp/config.json
index 74012ee..39bf74f 100644
--- a/hbp_nrp_virtual_coach/pynrp/config.json
+++ b/hbp_nrp_virtual_coach/pynrp/config.json
@@ -13,6 +13,9 @@
"storage-authentication": "authentication/authenticate",
"storage-experiment-list": "storage/experiments",
"csv-files": "experiment/%s/csvfiles",
+ "storage-models": "storage/models/%s/%s",
+ "storage-models-delete": "storage/models/%s/%s",
+ "storage-models-import": "storage/models/%s/%s",
"experiment-file": "storage/%s/%s",
"save-data": "experiment"
},
diff --git a/hbp_nrp_virtual_coach/pynrp/config.py b/hbp_nrp_virtual_coach/pynrp/config.py
index 0beab11..3eb6654 100644
--- a/hbp_nrp_virtual_coach/pynrp/config.py
+++ b/hbp_nrp_virtual_coach/pynrp/config.py
@@ -78,7 +78,9 @@ class Config(dict):
self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
'experiment-clone', 'experiment-delete',
'storage-authentication', 'storage-experiment-list',
- 'csv-files', 'experiment-file', 'experiment-import'])
+ 'csv-files', 'experiment-file', 'experiment-import',
+ 'storage-models', 'storage-models-delete',
+ 'storage-models-import'])
self.__validate('simulation-services', ['create', 'state', 'reset'])
self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
'sdf-world'])
diff --git a/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
index e49621c..7405371 100644
--- a/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
+++ b/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
@@ -128,6 +128,27 @@ class TestVirtualCoach(unittest.TestCase):
{'uuid': 'MockExperiment2_0', 'name': 'MockExperiment2_0'}]
+ self._mock_model_list = {'example_model_1': {"name":"example_model_1",
+ "displayName":"Example Model 1",
+ "type":"robots",
+ "isShared":"false",
+ "isCustom":"false",
+ "description":"Example Model 1 Description.",
+ "thumbnail":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA",
+ "path":"example_model_1",
+ "sdf":"example_model_1.sdf",
+ "configPath":"example_model_1/model.config"},
+ 'example_model_2': {"name":"example_model_2",
+ "displayName":"Example Model 2",
+ "type":"robots",
+ "isShared":"true",
+ "isCustom":"true",
+ "description":"Example Model 2 Description.",
+ "thumbnail":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA",
+ "path":"example_model_2",
+ "sdf":"example_model_2.sdf",
+ "configPath":"example_model_2/model.config"}}
+
def test_init_asserts_no_password(self):
# invalid environment
self.assertRaises(AssertionError, VirtualCoach, environment=True)
@@ -331,7 +352,6 @@ mock-server-5
@patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list')
@patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list')
def test_launch_no_available_servers(self, mock_list, servers_list):
-
# mock the Storage server call
mock_list.return_value = self._mock_exp_list
servers_list.return_value = []
@@ -600,7 +620,6 @@ mock-server-5
@patch('requests.post')
def test_set_experiment_list(self, request):
-
class Request(object):
status_code = 200
content = None
@@ -623,3 +642,43 @@ mock-server-5
path = os.path.join(self._tests_path, 'test_experiment_folder')
response = self._vc.import_experiment(path)
self.assertEqual(response.status_code, requests.codes.ok)
+
+
+ @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_model_list')
+ @patch('sys.stdout', new_callable=StringIO)
+ def test_print_available_models(self, mock_stdout, mock_list):
+ # invalid dev option (should be True or False; it is set to False by default)
+ self.assertRaises(AssertionError, self._vc.print_available_models, 23)
+
+ # mock the server call
+ mock_list.return_value = self._mock_model_list
+ self._vc.print_available_models(model_type='robots', user='all')
+
+ model_table = """
++--------------+--------------+--------------+--------------+--------------+
+| Name | Display Name | isShared | isCustom | Description |
++==============+==============+==============+==============+==============+
+| example_mode | Example | false | false | Example |
+| l_1 | Model 1 | | | Model 1 |
+| | | | | Description. |
++--------------+--------------+--------------+--------------+--------------+
+| example_mode | Example | true | true | Example |
+| l_2 | Model 2 | | | Model 2 |
+| | | | | Description. |
++--------------+--------------+--------------+--------------+--------------+
+ """
+ self.assertEqual(mock_stdout.getvalue().strip(), model_table.strip())
+
+ @patch('requests.post')
+ def test_import_model(self, mock_request):
+ mock_request.side_effect = [MagicMock(status_code=requests.codes.ok)]
+ self.assertRaises(Exception, self._vc.import_model, 'imaginary_file.zip')
+ path = os.path.join(self._tests_path, 'test.zip')
+ response = self._vc.import_model(path=path, model_type='robots')
+ self.assertEqual(response.status_code, requests.codes.ok)
+
+
+ @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_model_list')
+ def test_delete_model(self, mock_list):
+ mock_list.return_value = self._mock_model_list
+ self.assertRaises(ValueError, self._vc.delete_model, 'foo_name', 'foo_type')
diff --git a/hbp_nrp_virtual_coach/pynrp/virtual_coach.py b/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
index 7b8e388..9914902 100644
--- a/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
+++ b/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
@@ -132,7 +132,6 @@ class VirtualCoach(object):
# if the config is valid and the login doesn't fail, we're ready
logger.info('Ready.')
-
def __get_oidc_token(self, user_name, password):
"""
Attempts to acquire a oidc server token based on the provided credentials
@@ -697,6 +696,110 @@ class VirtualCoach(object):
% (response.status_code, response))
return response
+ def print_available_models(self, model_type, user="all"):
+ """
+ Prints available storage models.
+ :param model_type: type of the models to be shown ['environments', 'robots', 'brains']
+ :param user: username of models owner to be shown
+ """
+ assert isinstance(model_type, string_types)
+ assert isinstance(user, string_types)
+
+ if model_type not in ['robots', 'brains', 'environments']:
+ raise ValueError("Type must be a string in \
+ ['robots', 'brains', 'environments']")
+
+ model_list = self.__get_model_list(model_type=model_type, user=user)
+
+ table = Texttable()
+ table.header(['Name', 'Display Name',
+ 'isShared', 'isCustom', 'Description'])
+ for name, v in sorted(iter(model_list.items()), key=lambda x: x[1]['name']):
+ table.add_row([name, v['displayName'], v['isShared'],
+ v['isCustom'], v['description']])
+
+ # display the table
+ print(table.draw())
+
+ def import_model(self, path, model_type):
+ """
+ Imports a model (brain, robot or experiment as folder or zipped
+ folder) into user storage.
+ :param path: Path to the model folder or .zip file to be imported.
+ :param model_type: Model type to be imported ['environments', 'robots', 'brains']
+ """
+
+ assert isinstance(path, string_types)
+ assert isinstance(model_type, string_types)
+
+ if model_type not in ['robots', 'brains', 'environments']:
+ raise ValueError("Type must be a string in ['robots', \
+ 'brains', 'environments']")
+
+ if not os.path.isfile(path) and not os.path.isdir(path):
+ raise ValueError('The file or folder named %(path)s\
+ does not exist.' % {'path': path})
+
+ if os.path.isdir(path):
+ # Handles a model folder
+ content = self.__get_directory_content(path)
+ else:
+ # Handles a zip file
+ try:
+ with open(path, 'rb') as f:
+ content = f.read()
+ except Exception as e:
+ logger.error('The file %s could not be open', path)
+ raise e
+
+ file_headers = copy(self.__http_headers)
+ file_headers['Content-Type'] = 'application/octet-stream'
+ response = requests.post(self.__config['proxy-services']['storage-models-import'] %
+ (model_type, os.path.basename(path)),
+ data=content, headers=file_headers)
+
+ if response.status_code != requests.codes.ok:
+ raise Exception('Error when importing model: %d. Error: %s'
+ % (response.status_code, response))
+
+ return response
+
+ def delete_model(self, model_type, name):
+ """
+ Deletes a model from user storage.
+ :param name: Name of the model to be deleted.
+ :param model_type: Model type to be deleted ['environments', 'robots', 'brains']
+ """
+ assert isinstance(model_type, string_types)
+ assert isinstance(name, string_types)
+
+ if model_type not in ['robots', 'brains', 'environments']:
+ raise ValueError("Type must be a string in ['robots',\
+ 'brains', 'environments']")
+
+ model_list = self.__get_model_list(model_type)
+ if name not in model_list:
+ raise ValueError('Model Name: "%s" is invalid, the model does not exist in your'
+ ' storage. Please check the list of all models: \n%s'
+ % (name, model_list.keys()))
+ self.__http_client.delete(self.__config['proxy-services']['storage-models-delete']
+ % (model_type, name,), body={})
+ logger.info('Model "%s" deleted successfully', name)
+
+ def __get_model_list(self, model_type, user="all"):
+ """
+ Internal helper to retrieve and parse the model list from the backend proxy.
+
+ :param model_type: type of the model ['environments', 'robots', 'brains']
+ :param user: username of model owner to be printed
+ """
+ response = requests.get(self.__config['proxy-services']['storage-models']
+ % (user, model_type,), headers=self.__http_headers)
+
+ model_list = {model['name']: model for model in json.loads(response.content)}
+
+ return model_list
+
@staticmethod
def __zip_directory(dirpath, zip_filehandle):
"""
@@ -720,7 +823,7 @@ class VirtualCoach(object):
Internal helper function
It zips the target folder and returns its content
- :param dirpath: path to the experiment folder to be zipped
+ :param dirpath: path to the experiment/model folder to be zipped
"""
temp = tempfile.mktemp()
zip_file = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)
@@ -730,6 +833,7 @@ class VirtualCoach(object):
logger.error('The folder %s could not be zipped', dirpath)
raise e
zip_file.close()
- content = open(temp, 'rb').read()
+ with open(temp, 'rb') as f:
+ content = f.read()
os.remove(temp)
return content
--
GitLab