Merged in NRRPLT-8412-virtual-coach-models (pull request #37)

NRRPLT-8412-virtual-coach-models

* [NRRPLT-8412] Print, delete and import models Virtual Coach functions + tests

* [NRRPLT-8412] Fix pylint errors

* [NRRPLT-8412] remove redundant keyword arg

* [NRRPLT-8412] fix error unclosed file

* [NRRPLT-8412] fix error unclosed file in import model

* [NRRPLT-8412] fix type variable and condition

* [NRRPLT-8412] Fix failing tests and pycodestyle

Change-Id: Id7b95518ed4270468c798a76ae47e8cf740626e2


Approved-by: Eloy Retamino
Approved-by: Vahid Zolfaghari

hbp_nrp_virtual_coach/pynrp/config.json

@@ -13,6 +13,9 @@
         "storage-authentication": "authentication/authenticate",
         "storage-experiment-list": "storage/experiments",
         "csv-files": "experiment/%s/csvfiles",
+        "storage-models": "storage/models/%s/%s",
+        "storage-models-delete": "storage/models/%s/%s",
+        "storage-models-import": "storage/models/%s/%s",
         "experiment-file": "storage/%s/%s",
         "save-data": "experiment"
     },

hbp_nrp_virtual_coach/pynrp/config.py

@@ -78,7 +78,9 @@ class Config(dict):
         self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
                                            'experiment-clone', 'experiment-delete',
                                            'storage-authentication', 'storage-experiment-list',
-                                           'csv-files', 'experiment-file', 'experiment-import'])
+                                           'csv-files', 'experiment-file', 'experiment-import',
+                                           'storage-models', 'storage-models-delete',
+                                           'storage-models-import'])
         self.__validate('simulation-services', ['create', 'state', 'reset'])
         self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
                                                'sdf-world'])

hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py

@@ -128,6 +128,27 @@ class TestVirtualCoach(unittest.TestCase):
                                       {'uuid': 'MockExperiment2_0', 'name': 'MockExperiment2_0'}]
 
 
+        self._mock_model_list = {'example_model_1':  {"name":"example_model_1",
+                                                      "displayName":"Example Model 1",
+                                                      "type":"robots",
+                                                      "isShared":"false",
+                                                      "isCustom":"false",
+                                                      "description":"Example Model 1 Description.",
+                                                      "thumbnail":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA",
+                                                      "path":"example_model_1",
+                                                      "sdf":"example_model_1.sdf",
+                                                      "configPath":"example_model_1/model.config"},
+                                 'example_model_2':  {"name":"example_model_2",
+                                                      "displayName":"Example Model 2",
+                                                      "type":"robots",
+                                                      "isShared":"true",
+                                                      "isCustom":"true",
+                                                      "description":"Example Model 2 Description.",
+                                                      "thumbnail":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA",
+                                                      "path":"example_model_2",
+                                                      "sdf":"example_model_2.sdf",
+                                                      "configPath":"example_model_2/model.config"}}
+
     def test_init_asserts_no_password(self):
         # invalid environment
         self.assertRaises(AssertionError, VirtualCoach, environment=True)
@@ -331,7 +352,6 @@ mock-server-5
     @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_available_server_list')
     @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_experiment_list')
     def test_launch_no_available_servers(self, mock_list, servers_list):
-
         # mock the Storage server call
         mock_list.return_value = self._mock_exp_list
         servers_list.return_value = []
@@ -600,7 +620,6 @@ mock-server-5
 
     @patch('requests.post')
     def test_set_experiment_list(self, request):
-
         class Request(object):
             status_code = 200
             content = None
@@ -623,3 +642,43 @@ mock-server-5
         path = os.path.join(self._tests_path, 'test_experiment_folder')
         response = self._vc.import_experiment(path)
         self.assertEqual(response.status_code, requests.codes.ok)
+
+
+    @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_model_list')
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_print_available_models(self, mock_stdout, mock_list):
+        # invalid dev option (should be True or False; it is set to False by default)
+        self.assertRaises(AssertionError, self._vc.print_available_models, 23)
+
+        # mock the server call
+        mock_list.return_value = self._mock_model_list
+        self._vc.print_available_models(model_type='robots', user='all')
+
+        model_table = """
++--------------+--------------+--------------+--------------+--------------+
+|     Name     | Display Name |   isShared   |   isCustom   | Description  |
++==============+==============+==============+==============+==============+
+| example_mode | Example      | false        | false        | Example      |
+| l_1          | Model 1      |              |              | Model 1      |
+|              |              |              |              | Description. |
++--------------+--------------+--------------+--------------+--------------+
+| example_mode | Example      | true         | true         | Example      |
+| l_2          | Model 2      |              |              | Model 2      |
+|              |              |              |              | Description. |
++--------------+--------------+--------------+--------------+--------------+
+        """
+        self.assertEqual(mock_stdout.getvalue().strip(), model_table.strip())
+
+    @patch('requests.post')
+    def test_import_model(self, mock_request):
+        mock_request.side_effect = [MagicMock(status_code=requests.codes.ok)]
+        self.assertRaises(Exception, self._vc.import_model, 'imaginary_file.zip')
+        path = os.path.join(self._tests_path, 'test.zip')
+        response = self._vc.import_model(path=path, model_type='robots')
+        self.assertEqual(response.status_code, requests.codes.ok)
+
+
+    @patch('pynrp.virtual_coach.VirtualCoach._VirtualCoach__get_model_list')
+    def test_delete_model(self, mock_list):
+        mock_list.return_value = self._mock_model_list
+        self.assertRaises(ValueError, self._vc.delete_model,  'foo_name', 'foo_type')

hbp_nrp_virtual_coach/pynrp/virtual_coach.py

@@ -132,7 +132,6 @@ class VirtualCoach(object):
         # if the config is valid and the login doesn't fail, we're ready
         logger.info('Ready.')
 
-
     def __get_oidc_token(self, user_name, password):
         """
         Attempts to acquire a oidc server token based on the provided credentials
@@ -697,6 +696,110 @@ class VirtualCoach(object):
                             % (response.status_code, response))
         return response
 
+    def print_available_models(self, model_type, user="all"):
+        """
+        Prints available storage models.
+        :param model_type: type of the models to be shown ['environments', 'robots', 'brains']
+        :param user: username of models owner to be shown
+        """
+        assert isinstance(model_type, string_types)
+        assert isinstance(user, string_types)
+
+        if model_type not in ['robots', 'brains', 'environments']:
+            raise ValueError("Type must be a string in \
+                             ['robots', 'brains', 'environments']")
+
+        model_list = self.__get_model_list(model_type=model_type, user=user)
+
+        table = Texttable()
+        table.header(['Name', 'Display Name',
+                      'isShared', 'isCustom', 'Description'])
+        for name, v in sorted(iter(model_list.items()), key=lambda x: x[1]['name']):
+            table.add_row([name, v['displayName'], v['isShared'],
+                           v['isCustom'], v['description']])
+
+        # display the table
+        print(table.draw())
+
+    def import_model(self, path, model_type):
+        """
+        Imports a model (brain, robot or experiment as folder or zipped
+        folder) into user storage.
+        :param path: Path to the model folder or .zip file to be imported.
+        :param model_type: Model type to be imported ['environments', 'robots', 'brains']
+        """
+
+        assert isinstance(path, string_types)
+        assert isinstance(model_type, string_types)
+
+        if model_type not in ['robots', 'brains', 'environments']:
+            raise ValueError("Type must be a string in ['robots', \
+                             'brains', 'environments']")
+
+        if not os.path.isfile(path) and not os.path.isdir(path):
+            raise ValueError('The file or folder named %(path)s\
+                             does not exist.' % {'path': path})
+
+        if os.path.isdir(path):
+            # Handles a model folder
+            content = self.__get_directory_content(path)
+        else:
+            # Handles a zip file
+            try:
+                with open(path, 'rb') as f:
+                    content = f.read()
+            except Exception as e:
+                logger.error('The file %s could not be open', path)
+                raise e
+
+        file_headers = copy(self.__http_headers)
+        file_headers['Content-Type'] = 'application/octet-stream'
+        response = requests.post(self.__config['proxy-services']['storage-models-import'] %
+                                 (model_type, os.path.basename(path)),
+                                 data=content, headers=file_headers)
+
+        if response.status_code != requests.codes.ok:
+            raise Exception('Error when importing model: %d. Error: %s'
+                            % (response.status_code, response))
+
+        return response
+
+    def delete_model(self, model_type, name):
+        """
+        Deletes a model from user storage.
+        :param name: Name of the model to be deleted.
+        :param model_type: Model type to be deleted ['environments', 'robots', 'brains']
+        """
+        assert isinstance(model_type, string_types)
+        assert isinstance(name, string_types)
+
+        if model_type not in ['robots', 'brains', 'environments']:
+            raise ValueError("Type must be a string in ['robots',\
+                             'brains', 'environments']")
+
+        model_list = self.__get_model_list(model_type)
+        if name not in model_list:
+            raise ValueError('Model Name: "%s" is invalid, the model does not exist in your'
+                             ' storage. Please check the list of all models: \n%s'
+                             % (name, model_list.keys()))
+        self.__http_client.delete(self.__config['proxy-services']['storage-models-delete']
+                                  % (model_type, name,), body={})
+        logger.info('Model "%s" deleted successfully', name)
+
+    def __get_model_list(self, model_type, user="all"):
+        """
+        Internal helper to retrieve and parse the model list from the backend proxy.
+
+        :param model_type: type of the model ['environments', 'robots', 'brains']
+        :param user: username of model owner to be printed
+        """
+        response = requests.get(self.__config['proxy-services']['storage-models']
+                                % (user, model_type,), headers=self.__http_headers)
+
+        model_list = {model['name']: model for model in json.loads(response.content)}
+
+        return model_list
+
     @staticmethod
     def __zip_directory(dirpath, zip_filehandle):
         """
@@ -720,7 +823,7 @@ class VirtualCoach(object):
         Internal helper function
         It zips the target folder and returns its content
 
-        :param dirpath: path to the experiment folder to be zipped
+        :param dirpath: path to the experiment/model folder to be zipped
         """
         temp = tempfile.mktemp()
         zip_file = zipfile.ZipFile(temp, 'w', zipfile.ZIP_DEFLATED)
@@ -730,6 +833,7 @@ class VirtualCoach(object):
             logger.error('The folder %s could not be zipped', dirpath)
             raise e
         zip_file.close()
-        content = open(temp, 'rb').read()
+        with open(temp, 'rb') as f:
+            content = f.read()
         os.remove(temp)
         return content