diff --git a/hbp_nrp_virtual_coach/doc/source/tutorials/interacting_exp.rst b/hbp_nrp_virtual_coach/doc/source/tutorials/interacting_exp.rst
index cdb44fe280a21394421ec7613f8877a9f31d6043..ba14836a61b1520aff8f3e5c9ffe1813ca9b1c88 100644
--- a/hbp_nrp_virtual_coach/doc/source/tutorials/interacting_exp.rst
+++ b/hbp_nrp_virtual_coach/doc/source/tutorials/interacting_exp.rst
@@ -15,7 +15,7 @@ When an experiment is launched, it is initially in the *paused* state. This mean
sim.get_state()
-The state of the simulation is returned as a string. In this case calling ``get_state()`` will return *paused*. The possible states we can set a simulation to are *paused*, *started* and *stopped*. We can alternatively start and pause the simulation by calling:
+The state of the simulation is returned as a string. In this case calling ``get_state()`` will return *paused*. The possible states we can set a simulation to are *paused*, *started* and *stopped*. We can alternatively start and pause the simulation by calling:
.. code-block:: python
@@ -118,7 +118,7 @@ We can also add new transfer functions. For this we only need to provide the tra
recorder.record_entry(joint_state.value.name[i], t, joint_state.value.position[i])"""
sim.add_transfer_function(csv_joint_state_monitor)
-
+
.. code-block:: python
csv_robot_position = """@nrp.MapCSVRecorder("recorder", filename="robot_position.csv", headers=["x", "y", "z"])
@@ -153,20 +153,20 @@ We can also add new transfer functions. For this we only need to provide the tra
Those transfer functions will log the simulation time to the log console every two seconds.
-Getting CSV Data
+Getting Experiment Data
^^^^^^^^^^^^^^^^
-With the transfer functions that we wrote, we can access all csv data from the Virtual Coach and plot or analyze the data. To know what kind of data is being saved to csv files in an experiment, you can print out the names of the csv files first using this call:
+With the transfer functions that we wrote, we can access all experiment data from the Virtual Coach and plot or analyze the data. You can specify, if you want to inspect 'csv' data or 'profiler' data, first is created with Transfer Function recording, latter from the NRP cle profiler. To know what kind of data is being saved to files in an experiment, you can print out the names of the files first using this call:
.. code-block:: python
- vc.print_csv_last_run_files(exp_id)
+ vc.print_last_run_files(exp_id, 'csv')
-In the case of the Template Husky experiment, this will print out *all_spikes.csv* and *all_joints_positions.csv* and *robot_position.csv*. We can now get the data from any one of these files. Note that these files will be populated only if a simulation has been running. Here we will get and print out the Spike data:
+In the case of the Template Husky experiment, this will print out *all_spikes.csv* and *all_joints_positions.csv* and *robot_position.csv*. We can now get the data from any one of these files. Note that these files will be populated only if a simulation has been running. Here we will get the Spike data:
.. code-block:: python
- spikes = vc.print_csv_last_run_file(exp_id, 'all_spikes.csv')
+ spikes = vc.get_last_run_file(exp_id, 'csv', 'all_spikes.csv')
print(spikes)
[[u'id', u'time', u'Simulation_reset'],
[u'3.0', u'0.10000000000000001', u'RESET'],
@@ -233,4 +233,3 @@ It is also possible to reset certain aspects of the simulation from the Virtual
sim.reset('full')
If you want to look at more concrete example experiments run from the Virtual Coach, you can check out the **hbp_nrp_virtual_coach/examples** directory.
-
diff --git a/hbp_nrp_virtual_coach/pynrp/config.json b/hbp_nrp_virtual_coach/pynrp/config.json
index 39bf74fad48761b62570d515bef4cf38360b0d4b..084b021e2c535458ccd42073a95d335ea8f541be 100644
--- a/hbp_nrp_virtual_coach/pynrp/config.json
+++ b/hbp_nrp_virtual_coach/pynrp/config.json
@@ -12,10 +12,10 @@
"experiment-import": "storage/importExperiment",
"storage-authentication": "authentication/authenticate",
"storage-experiment-list": "storage/experiments",
- "csv-files": "experiment/%s/csvfiles",
"storage-models": "storage/models/%s/%s",
"storage-models-delete": "storage/models/%s/%s",
"storage-models-import": "storage/models/%s/%s",
+ "files": "experiment/%s/files/%s",
"experiment-file": "storage/%s/%s",
"save-data": "experiment"
},
diff --git a/hbp_nrp_virtual_coach/pynrp/config.py b/hbp_nrp_virtual_coach/pynrp/config.py
index 3eb6654fff7f4862e77322efeff2bd72b2707d43..a363bb3561cfd4053b8afaed933a59db316e90ca 100644
--- a/hbp_nrp_virtual_coach/pynrp/config.py
+++ b/hbp_nrp_virtual_coach/pynrp/config.py
@@ -78,9 +78,9 @@ class Config(dict):
self.__validate('proxy-services', ['experiment-list', 'available-servers', 'server-info',
'experiment-clone', 'experiment-delete',
'storage-authentication', 'storage-experiment-list',
- 'csv-files', 'experiment-file', 'experiment-import',
'storage-models', 'storage-models-delete',
- 'storage-models-import'])
+ 'storage-models-import',
+ 'files', 'experiment-file', 'experiment-import'])
self.__validate('simulation-services', ['create', 'state', 'reset'])
self.__validate('simulation-scripts', ['state-machine', 'transfer-function', 'brain',
'sdf-world'])
diff --git a/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
index c5fef8b4cca7ce3c80ca18439c6748c920364e2c..d4cc2a9df225a721b9d16735ca088b93e20d933e 100644
--- a/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
+++ b/hbp_nrp_virtual_coach/pynrp/tests/test_virtual_coach.py
@@ -538,10 +538,10 @@ mock-server-5
@patch('requests.get')
@patch('sys.stdout', new_callable=StringIO)
- def test_print_experiment_runs_csv(self, mock_stdout,mock_request):
+ def test_print_experiment_runs(self, mock_stdout,mock_request):
mock_request.return_value = self.__mock_csv_files_response()
- content = self._vc.print_csv_experiment_runs('exp_id')
+ content = self._vc.print_experiment_runs_files('exp_id', 'csv')
csv_runs = """
+--------+---------+-------+
| Run id | Date | Bytes |
@@ -555,10 +555,10 @@ mock-server-5
@patch('requests.get')
@patch('sys.stdout', new_callable=StringIO)
- def test_print_experiment_run_files_csv(self, mock_stdout,mock_request):
+ def test_print_experiment_run_files(self, mock_stdout,mock_request):
mock_request.return_value = self.__mock_csv_files_response()
- self._vc.print_csv_experiment_run_files('exp_id', 0)
+ self._vc.print_experiment_run_files('exp_id', 'csv', 0)
csv_run_files = """
+------+------+
| File | Size |
@@ -577,7 +577,7 @@ mock-server-5
def test_print_last_run_files_csv(self, mock_stdout,mock_request):
mock_request.return_value = self.__mock_csv_files_response()
- self._vc.print_csv_last_run_files('exp_id')
+ self._vc.print_last_run_files('exp_id', 'csv')
csv_run_files = """
+------+------+
| File | Size |
@@ -600,11 +600,11 @@ mock-server-5
self.__mock_csv_files_response(),
self.__mock_csv_file_response()
]
- self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
- csv_content = self._vc.get_csv_experiment_run_file('exp_id', 0, 'csv1')
+ self._vc._VirtualCoach__get_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_file_content)
+ csv_content = self._vc.get_experiment_run_file('exp_id', 'csv', 0, 'csv1')
self.assertEqual(csv_content, 'a,b,c\n1,2,3')
- self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid1')
+ self._vc._VirtualCoach__get_file_content.assert_called_once_with('exp_id', u'uuid1')
@patch('requests.get')
def test_get_last_run_file_csv(self, mock_request):
@@ -613,10 +613,10 @@ mock-server-5
self.__mock_csv_file_response()
]
- self._vc._VirtualCoach__get_csv_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_csv_file_content)
- csv_content = self._vc.get_csv_last_run_file('exp_id', 'csv1')
+ self._vc._VirtualCoach__get_file_content = MagicMock(side_effect=self._vc._VirtualCoach__get_file_content)
+ csv_content = self._vc.get_last_run_file('exp_id', 'csv', 'csv1')
self.assertEqual(csv_content, 'a,b,c\n1,2,3')
- self._vc._VirtualCoach__get_csv_file_content.assert_called_once_with('exp_id', u'uuid4')
+ self._vc._VirtualCoach__get_file_content.assert_called_once_with('exp_id', u'uuid4')
@patch('requests.post')
def test_set_experiment_list(self, request):
diff --git a/hbp_nrp_virtual_coach/pynrp/virtual_coach.py b/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
index da450c94af8cb8e8e30e11b63f1fe90c11ebf02d..afb04d2b23a419b2c4e507df43fd54e5e1b1976c 100644
--- a/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
+++ b/hbp_nrp_virtual_coach/pynrp/virtual_coach.py
@@ -182,7 +182,7 @@ class VirtualCoach(object):
# the user is forwarded to the approve page if not approved yet
if 'access_token' not in urlparse.urlparse(res.geturl()).fragment.lower():
raise Exception('Login to NRP online and give your consent. Then return to VC.')
-
+
url_with_fragment = res.geturl()
# parse and return token
@@ -495,149 +495,155 @@ class VirtualCoach(object):
logger.info('Experiment "%s" cloned successfully', experiment_id)
return res.content
- def print_csv_experiment_runs(self, exp_id):
+ def print_experiment_runs_files(self, exp_id, file_type):
"""
- Prints the list of ids of experiment runs that generated CSV files. The ids can be used
- to retrieve the CSV data
+ Prints the list of ids of experiment runs that generated files. The ids can be used
+ to retrieve the data
- :param exp_id: The experiment id for which to retrieve the list of CSV simulation runs
+ :param exp_id: The experiment id for which to retrieve the list of simulation runs
+ :param file_type: type of files ['csv', 'profiler']
"""
- csv_files = self.__get_available_CSV_files(exp_id)
+ files = self.__get_available_files(exp_id, file_type)
table = Texttable()
table.header(['Run id', 'Date', 'Bytes'])
table.set_cols_align(['r', 'c', 'r'])
- for i, run_date in enumerate(sorted(csv_files.keys())):
- run_size = sum(file['size'] for file in list(csv_files[run_date].values()))
+ for i, run_date in enumerate(sorted(files.keys())):
+ run_size = sum(file['size'] for file in list(files[run_date].values()))
table.add_row([i, run_date, run_size])
logger.info('List of simulation runs')
print(table.draw())
- def print_csv_experiment_run_files(self, exp_id, run_id):
+ def print_experiment_run_files(self, exp_id, file_type, run_id):
"""
- Prints the list of CSV files for a given experiment run
+ Prints the list of files for a given experiment run
- :param exp_id: The experiment id for which to retrieve the list of CSV files
- :param run_id: The run id for which to retrieve the list of CSV files
+ :param exp_id: The experiment id for which to retrieve the list of files
+ :param file_type: type of files ['csv', 'profiler']
+ :param run_id: The run id for which to retrieve the list of files
"""
- csv_files = self.__get_available_CSV_files(exp_id)
+ files = self.__get_available_files(exp_id, file_type)
table = Texttable()
table.header(['File', 'Size'])
table.set_cols_align(['l', 'r'])
- sorted_runs = sorted(csv_files.keys())
+ sorted_runs = sorted(files.keys())
if not 0 <= run_id < len(sorted_runs):
raise Exception('Could not find run %i, %i runs were found' %
(run_id, len(sorted_runs)))
- for csv_file in list(csv_files[sorted_runs[run_id]].values()):
- table.add_row([csv_file['name'], csv_file['size']])
+ for file in list(files[sorted_runs[run_id]].values()):
+ table.add_row([file['name'], file['size']])
logger.info('Run %i list of files.', run_id)
print(table.draw())
- def print_csv_last_run_files(self, exp_id):
+ def print_last_run_files(self, exp_id, file_type):
"""
- Prints the list of CSV files for the last run of an experiment
+ Prints the list of files for the last run of an experiment
- :param exp_id: The experiment id for which to retrieve the list of CSV files
+ :param exp_id: The experiment id for which to retrieve the list of files
+ :param file_type: type of files ['csv', 'profiler']
"""
- csv_files = self.__get_available_CSV_files(exp_id)
+ files = self.__get_available_files(exp_id, file_type)
table = Texttable()
table.header(['File', 'Size'])
table.set_cols_align(['l', 'r'])
- sorted_runs = sorted(csv_files.keys())
+ sorted_runs = sorted(files.keys())
if not sorted_runs:
raise Exception('Could not find any run')
- for csv_file in list(csv_files[sorted_runs[-1]].values()):
- table.add_row([csv_file['name'], csv_file['size']])
+ for file in list(files[sorted_runs[-1]].values()):
+ table.add_row([file['name'], file['size']])
logger.info('Last run list of files')
print(table.draw())
- def get_csv_experiment_run_file(self, exp_id, run_id, file_name):
+ def get_experiment_run_file(self, exp_id, file_type, run_id, file_name):
"""
- Retrieves a CSV file content
+ Retrieves a file content
:param exp_id: The experiment id
+ :param file_type: type of files ['csv', 'profiler']
:param run_id: The run id
:param file_uuid: The file uuid
"""
- csv_files = self.__get_available_CSV_files(exp_id)
- sorted_runs = sorted(csv_files.keys())
+ files = self.__get_available_files(exp_id, file_type)
+ sorted_runs = sorted(files.keys())
if not 0 <= run_id < len(sorted_runs):
raise Exception('Could not find run %i, %i runs were found' %
(run_id, len(sorted_runs)))
- if file_name not in csv_files[sorted_runs[run_id]]:
- file_names = ', '.join(f['name'] for f in list(csv_files[sorted_runs[run_id]].values()))
+ if file_name not in files[sorted_runs[run_id]]:
+ file_names = ', '.join(f['name'] for f in list(files[sorted_runs[run_id]].values()))
raise Exception('Could not find file \'%s\' in run %i, available file names are: %s' %
(file_name, run_id, file_names))
- file_uuid = csv_files[sorted_runs[run_id]][file_name]['uuid']
- return self.__get_csv_file_content(exp_id, file_uuid)
+ file_uuid = files[sorted_runs[run_id]][file_name]['uuid']
+ return self.__get_file_content(exp_id, file_uuid)
- def get_csv_last_run_file(self, exp_id, file_name):
+ def get_last_run_file(self, exp_id, file_type, file_name):
"""
- Retrieves a CSV file content for the last run
+ Retrieves a file content for the last run
:param exp_id: The experiment id
+ :param file_type: type of files ['csv', 'profiler']
:param file_name: The file name
"""
- csv_files = self.__get_available_CSV_files(exp_id)
- sorted_runs = sorted(csv_files.keys())
+ files = self.__get_available_files(exp_id, file_type)
+ sorted_runs = sorted(files.keys())
if not sorted_runs:
raise Exception('Could not find any run')
- if file_name not in csv_files[sorted_runs[-1]]:
+ if file_name not in files[sorted_runs[-1]]:
file_names = ', '.join(
- file['name'] for file in list(csv_files[sorted_runs[-1]].values()))
+ file['name'] for file in list(files[sorted_runs[-1]].values()))
raise Exception('Could not find file \'%s\' in last run, available file names are: %s' %
(file_name, file_names))
- file_uuid = csv_files[sorted_runs[-1]][file_name]['uuid']
- return self.__get_csv_file_content(exp_id, file_uuid)
+ file_uuid = files[sorted_runs[-1]][file_name]['uuid']
+ return self.__get_file_content(exp_id, file_uuid)
- def __get_available_CSV_files(self, experiment_id):
+ def __get_available_files(self, experiment_id, file_type):
"""
- Internal helper to retrieve the list of CSV files available for an experiment
+ Internal helper to retrieve the list of files available for an experiment
- :param experiment_id: The experiment id for which to retrieve the list of CSV files
+ :param experiment_id: The experiment id for which to retrieve the list of files
+ :param file_type: type of files ['csv', 'profiler']
"""
- response = requests.get(self.__config['proxy-services']['csv-files'] % (experiment_id,),
- headers=self.__http_headers)
+ response = requests.get(self.__config['proxy-services']['files'] % \
+ (experiment_id, file_type,), headers=self.__http_headers)
if response.status_code != http.client.OK:
- raise Exception('Error when getting CSV files Status Code: %d. Error: %s'
+ raise Exception('Error when getting files Status Code: %d. Error: %s'
% (response.status_code, response))
- csv_files = json.loads(response.content)
+ files = json.loads(response.content)
distinct_runs = defaultdict(dict)
- for csv_file in csv_files:
- distinct_runs[csv_file['folder']][csv_file['name']] = csv_file
+ for file in files:
+ distinct_runs[file['folder']][file['name']] = file
return distinct_runs
- def __get_csv_file_content(self, exp_id, file_uuid):
+ def __get_file_content(self, exp_id, file_uuid):
"""
- Internal helper method to retrieve a CSV file content
+ Internal helper method to retrieve a file content
- :param exp_id: The experiment id for which to retrieve the CSV file content
+ :param exp_id: The experiment id for which to retrieve the file content
:param file_uuid: The file uuid for which to retrieve the content
"""
- logger.info('Retrieving CSV file.')
+ logger.info('Retrieving file.')
response = requests.get(self.__config['proxy-services']['experiment-file'] %
(exp_id, file_uuid),
headers=self.__http_headers)
if response.status_code != http.client.OK:
- raise Exception('Error when getting CSV file Status Code: %d. Error: %s'
+ raise Exception('Error when getting file Status Code: %d. Error: %s'
% (response.status_code, response))
return response.content