Merged in NRRPLT-6660 (pull request #5)

[NRRPLT-6660] Removed all references to CSV saving as everything is done in th backend

Approved-by: Susie Murphy <susie.murphy@epfl.ch>
Approved-by: Kepa Cantero <cantero@fortiss.org>

examples/integration_test/it.py

@@ -110,7 +110,7 @@ def run(oidc_username, storage_username):
 
     # running array of test case results, unfortunately we have to hardcode the number of
     # test cases because the indeterminate progress bar is not helpful for the tester
-    NUM_TEST_CASES = 31
+    NUM_TEST_CASES = 29
     results = TestCases(NUM_TEST_CASES)
 
     try:
@@ -399,22 +399,6 @@ def tf(t):
             raise TestCaseError('Population step update did not resume running simulation.')
         results.done(True)
 
-        ##
-        ## Recorded CSV Data Interaction
-        ##
-
-        results.start('Retrieving Available CSV Files')
-        csvs = sim._Simulation__get_csv_file_names()
-        if len(csvs) == 0:
-            raise TestCaseError('No CSV filenames returned for experiment.')
-        results.done(True)
-
-        results.start('Retrieving A CSV File')
-        csv_data = sim.get_csv_data(csvs[0])
-        if len(csv_data) == 0:
-            raise TestCaseError('No CSV data returned.')
-        results.done(True)
-
         ##
         ## Reset Different Simulation Parts
         ##

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py

@@ -86,7 +86,7 @@ class Config(dict):
         self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
                                            'experiment-clone', 'experiment-delete',
                                            'storage-authentication', 'storage-experiment-list'])
-        self.__validate('simulation-services', ['create', 'state', 'reset', 'csv-recorders'])
+        self.__validate('simulation-services', ['create', 'state', 'reset'])
         self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
                                                'sdf-world'])
         self.__validate('reset-services', ['robot_pose', 'full', 'world', 'brain'])

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py

@@ -594,26 +594,6 @@ class Simulation(object):
         self.__save_experiment_data('brain', {'data': pynn_script,
                                               'brain_populations': populations})
 
-    def save_csv(self):
-        """
-        Saves the recorded csv data to storage
-        """
-
-        if not self.__sim_url:
-            raise Exception("No simulation data!")
-
-        url = '%s/%s' % (self.__sim_url, self.__config['simulation-services']['csv-recorders'])
-
-        try:
-            status_code, _ = self.__http_client.put(url, body={})
-
-            if status_code != httplib.OK:
-                raise Exception('Error status code %s' % status_code)
-            self.__logger.info("Saved CSV data")
-        except Exception as err:
-            self.__logger.info(err)
-            raise Exception("Failed to save CSV.")
-
     def save_world(self):
         """
         Saves the current sdf world to the storage
@@ -757,57 +737,6 @@ class Simulation(object):
         assert isinstance(populations, dict)
         self.__set_brain(self.get_brain(), populations, 1)
 
-    def __get_all_csv_data(self):
-        """
-        Internal helper that returns the simulation's recorded data from csv files up to the point
-        it has been called.
-        """
-        if not self.__server or not self.__sim_url:
-            raise Exception("Simulation has not been created, cannot get csv data!")
-
-        self.__logger.info("Attempting to retrieve recorders' csv data")
-        url = '%s/%s' % (self.__sim_url, self.__config['simulation-services']['csv-recorders'])
-        status_code, content = self.__http_client.get(url)
-
-        if status_code != httplib.OK:
-            raise Exception("Unable to get simulation CSV data, HTTP status %s"
-                            % status_code)
-
-        return json.loads(content)
-
-    def __get_csv_file_names(self):
-        """
-        Internal helper to get all the csv file names the contain recorded simulation data.
-        """
-        return [recorder['file'] for recorder in self.__get_all_csv_data()]
-
-    def print_csv_file_names(self):
-        """
-        Prints a list of all csv file names that contain recorded simulation data.
-        """
-        print self.__get_csv_file_names()
-
-    def get_csv_data(self, file_name):
-        """
-        Returns the recorded csv data as a list, where each entry is a list of strings and
-        represents one row in the original csv file.
-
-        :param file_name: The name of the csv file
-        """
-
-        assert isinstance(file_name, (str, unicode))
-
-        csv_data = self.__get_all_csv_data()
-        csv_files = self.__get_csv_file_names()
-
-        if file_name not in csv_files:
-            raise ValueError("File '%s' does not exist. Please check the csv file names \n%s"
-                             % (file_name, '\n'.join(csv_files)))
-
-        recorder_data = next(rec['data'] for rec in csv_data if rec['file'] == file_name)
-
-        return [item.split(',') for item in recorder_data]
-
     def reset(self, reset_type):
         """
         Resets the simulation according to the type the user wants. Successful reset will pause the

hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py

@@ -491,23 +491,6 @@ class TestSimulation(unittest.TestCase):
             '%s/experiment/%s/sdf_world' % (serverurl, exp_id),
             body={})
 
-        self._sim._Simulation__http_client.put.reset_mock()
-        self._sim._Simulation__http_client.put = Mock(
-            return_value=(httplib.INTERNAL_SERVER_ERROR, None))
-        self.assertRaises(Exception, self._sim.save_csv)
-
-        self._sim._Simulation__http_client.put.reset_mock()
-        self._sim._Simulation__sim_url = 'url'
-        self.assertRaises(Exception, self._sim.save_csv)
-
-        self._sim._Simulation__http_client.put.reset_mock()
-        self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK, None))
-        self._sim.save_csv()
-
-        self._sim._Simulation__http_client.put.assert_called_once_with(
-            '%s/csv-recorders' % self._sim._Simulation__sim_url,
-            body={})
-
     @patch('sys.stdout', new_callable=StringIO)
     def test_print_scripts(self, mock_stdout):
         self._sim._Simulation__server = 'server'
@@ -612,60 +595,6 @@ class TestSimulation(unittest.TestCase):
         self.assertEqual(self._sim._Simulation__status_callbacks, [])
         self._sim._Simulation__logger.info.assert_called_with('Simulation has been stopped.')
 
-    def test_get_all_csv_data(self):
-        self._sim._Simulation__server = 'server'
-        self._sim._Simulation__sim_url = 'url'
-
-        # mock the oidc call
-        self._sim._Simulation__http_client.get = Mock()
-        self._sim._Simulation__http_client.get.return_value = (httplib.OK,
-                                                             '[{"data": [], "file": "spikes"}]')
-        files = self._sim._Simulation__get_csv_file_names()
-        self._sim._Simulation__http_client.get.assert_called_once()
-        self.assertEqual(files, ['spikes'])
-
-    def test_get_all_csv_data_failed(self):
-        self.assertRaises(Exception, self._sim._Simulation__get_all_csv_data)
-
-        self._sim._Simulation__server = 'server'
-        self._sim._Simulation__sim_url = 'url'
-
-        # mock the OIDC call
-        self._sim._Simulation__http_client.get = Mock()
-        self._sim._Simulation__http_client.get.return_value = (httplib.NOT_FOUND,
-                                                             None)
-
-        self.assertRaises(Exception, self._sim._Simulation__get_all_csv_data)
-        self._sim._Simulation__http_client.get.assert_called_once()
-
-    def test_get_csv_data(self):
-
-        self.assertRaises(AssertionError, self._sim.get_csv_data, None)
-
-        self._sim._Simulation__server = 'server'
-        self._sim._Simulation__sim_url = 'url'
-
-        self._sim._Simulation__http_client.get = Mock()
-        self._sim._Simulation__http_client.get.return_value = (httplib.OK,
-                                                             '[{"data": ["1,10.0", "2,15.0"],'
-                                                             '"file": "spikes"}]')
-
-        self.assertEqual(self._sim.get_csv_data('spikes'), [['1', '10.0'], ['2', '15.0']])
-        self.assertRaises(ValueError, self._sim.get_csv_data, 'joints')
-
-    @patch('sys.stdout', new_callable=StringIO)
-    def test_print_csv_file_names(self, mock_stdout):
-        self._sim._Simulation__server = 'server'
-        self._sim._Simulation__sim_url = 'url'
-
-        # mock the oidc call
-        self._sim._Simulation__http_client.get = Mock()
-        self._sim._Simulation__http_client.get.return_value = (httplib.OK,
-                                                             '[{"data": [], "file": "spikes"}]')
-        self._sim.print_csv_file_names()
-        self.assertEqual(mock_stdout.getvalue().strip(), "[u'spikes']")
-        self._sim._Simulation__http_client.get.assert_called_once()
-
     def test_reset(self):
         self._sim._Simulation__server = 'server'
         self._sim._Simulation__sim_url = 'url'