diff --git a/Makefile b/Makefile
index b379286b85fc33af3e5c1751bfe4bf7536df068a..b4d809e1f7b4bc80d2ff1d00e42e44a5ef281232 100644
--- a/Makefile
+++ b/Makefile
@@ -11,15 +11,17 @@ COVER_PACKAGES=hbp_nrp_virtual_coach
DOC_MODULES=hbp_nrp_virtual_coach/doc
DOC_REPO=--doc-repo ssh://bbpcode.epfl.ch/infra/jekylltest
-PYTHON_PIP_VERSION?=pip==9.0.3
+PYTHON_PIP_VERSION?='pip>=19'
##### DO NOT MODIFY BELOW #####################
+python_version_full := $(shell python -c "import sys; maj, min = sys.version_info[:2]; print('{}.{}'.format(maj, min))" 2>&1)
+
ifeq ($(NRP_INSTALL_MODE),user)
- $(shell cp -af $(HBP)/user-scripts/config_files/platform_venv/* $(VIRTUAL_ENV)/lib/python2.7/site-packages/ )
+ $(shell cp -af $(HBP)/user-scripts/config_files/platform_venv/* $(VIRTUAL_ENV)/lib/python$(python_version_full)/site-packages/ )
include user_makefile
else
- $(shell cp -af $(HBP)/user-scripts/config_files/platform_venv/* $(VIRTUAL_ENV)/lib/python2.7/site-packages/ )
+ $(shell cp -af $(HBP)/user-scripts/config_files/platform_venv/* $(VIRTUAL_ENV)/lib/python$(python_version_full)/site-packages/ )
CI_REPO?=git@bitbucket.org:hbpneurorobotics/admin-scripts.git
CI_DIR?=$(HBP)/admin-scripts/ContinuousIntegration
THIS_DIR:=$(PWD)
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index 54ea51d92095a82547eb541054ef2a81c53477aa..109868b344b3883a9cc1407ca314617806b1a09d 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -28,6 +28,9 @@ pipelines:
# Copy bbp-client from user-scripts (before make devinstall)
- cp -af $HBP/user-scripts/config_files/platform_venv/* $VIRTUAL_ENV_PATH/lib/python2.7/site-packages/
+ # Delete pip lock in VIRTUAL_ENV so to force checking for pip upgrade
+ - rm $VIRTUAL_ENV/new-pip.txt
+
# Build
- export IGNORE_LINT='platform_venv|config_files|examples/integration_test/test_experiment_folder'
- make verify_base || { if [ -f pylint.txt ]; then echo "----------"; echo "PYLINT.TXT"; echo "----------";cat pylint.txt; fi; if [ -f pep8.txt ]; then echo "----------"; echo "PEP8.TXT"; echo "----------";cat pep8.txt; fi; exit 1; }
diff --git a/examples/integration_test/it.py b/examples/integration_test/it.py
index acdac7be2cbec26ac6c00da4694208f3843de835..6f159e02d8209c11cd1d096555e032c183cb7630 100644
--- a/examples/integration_test/it.py
+++ b/examples/integration_test/it.py
@@ -25,12 +25,20 @@ platform (e.g. specific text values, timings, etc.), but SHOULD interact with al
interfaces/APIs that are supported.
"""
+# pylint: disable=W0622
+
+from __future__ import print_function
+
+from builtins import str
+from builtins import zip
+from builtins import object
import argparse
import logging
import json
import sys
import time
import traceback
+import os
# virtualcoach and platform_venv specific packages, ensure environment is correct
try:
@@ -38,14 +46,14 @@ try:
import progressbar
except ImportError:
- print
+ print()
print('Failed to import the VirtualCoach or access packages in the platform_venv! Aborting.')
- print
+ print()
print('Please make sure you have:')
- print
+ print()
print('\t1. run "make devinstall" in the VirtualCoach repo')
print('\t2. use "cle-virtual-coach it.py" to run this script')
- print
+ print()
sys.exit(-1)
##
@@ -117,7 +125,7 @@ def run(oidc_username, storage_username):
# the simulation we will launch, defined here so we have a cleanup reference at any point
sim = None
-
+ path = os.path.dirname(os.path.abspath(__file__))
##
## Server Information and Experiment List Interaction
##
@@ -137,7 +145,7 @@ def run(oidc_username, storage_username):
# ensure there is a running server that is not currently running an experiment
results.start('Checking For Available Backend')
available_servers = vc._VirtualCoach__get_available_server_list()
- if len(available_servers) == 0:
+ if not available_servers:
raise TestCaseError('No available backends to run test on.')
results.done(True)
@@ -145,7 +153,7 @@ def run(oidc_username, storage_username):
# the experiment first if not.
results.start('Cloning a new Empty Template Husky Experiment from the Storage Server')
- if 'ExDTemplateHusky' not in [s[0] for s in server_info.iteritems()]:
+ if 'ExDTemplateHusky' not in [s[0] for s in server_info.items()]:
raise TestCaseError('Husky Template Experiment is not available on the server to be'
' cloned.')
experiment_id = vc.clone_experiment_to_storage('ExDTemplateHusky')
@@ -179,12 +187,11 @@ def run(oidc_username, storage_username):
##
results.start('Importing an experiment folder')
- response = vc.import_experiment('test_experiment_folder')
+ response = vc.import_experiment(path + '/test_experiment_folder')
new_experiment_id = json.loads(response.text)['destFolderName']
if new_experiment_id not in vc._VirtualCoach__get_experiment_list(cloned=True):
raise TestCaseError('Importing an experiment folder failed')
- else:
- vc.delete_cloned_experiment(new_experiment_id)
+ vc.delete_cloned_experiment(new_experiment_id)
results.done(True)
##
@@ -192,12 +199,11 @@ def run(oidc_username, storage_username):
##
results.start('Importing a zipped experiment folder')
- response = vc.import_experiment('test_experiment_folder.zip')
+ response = vc.import_experiment(path + '/test_experiment_folder.zip')
new_experiment_id = json.loads(response.text)['destFolderName']
if new_experiment_id not in vc._VirtualCoach__get_experiment_list(cloned=True):
raise TestCaseError('Importing an experiment folder failed')
- else:
- vc.delete_cloned_experiment(new_experiment_id)
+ vc.delete_cloned_experiment(new_experiment_id)
results.done(True)
##
@@ -267,12 +273,12 @@ def run(oidc_username, storage_username):
# retrieve transfer functions
results.start('Retrieving Transfer Functions')
tfs = sim._Simulation__get_simulation_scripts('transfer-function')['data']
- if len(tfs) == 0:
+ if not tfs:
raise TestCaseError('No transfer functions returned for experiment.')
results.done(True)
# create a valid tf with a modified import and an invalid tf for testing
- tf_name = str(tfs.keys()[0])
+ tf_name = str(list(tfs.keys())[0])
valid_tf = 'import os\n' + tfs[tf_name]
invalid_tf = 'this is invalid!\n' + tfs[tf_name]
@@ -448,23 +454,23 @@ def tf(t):
results.done(True)
# handle test case or unexpected failures, attempt to cleanup and terminate
- except Exception, e:
+ except Exception as e:
# fail the last test case
results.abort()
# check if the test case has just failed
- print
+ print()
if isinstance(e, TestCaseError):
- print 'Test case failed: %s - attempting to cleanup.' % e
+ print('Test case failed: %s - attempting to cleanup.' % e)
# unhandled exception means a virtual coach failure, log it
else:
- print
- print 'Unexpected exception encountered during execution, attempting to cleanup.'
- print
+ print()
+ print('Unexpected exception encountered during execution, attempting to cleanup.')
+ print()
traceback.print_exc(e)
- print
+ print()
# if a sim is running, try to shut it down cleanly, we can't do much more
if sim:
@@ -508,29 +514,29 @@ if __name__ == '__main__':
logging.disable(logging.CRITICAL)
# banner with user warnings / prompts
- print ''.center(80, '=')
- print '|%s|' % ''.center(78, ' ')
- print '|%s|' % 'VirtualCoach Integration Test Script'.center(78, ' ')
- print '|%s|' % ''.center(78, ' ')
- print '|%s|' % ''.center(78, ' ')
+ print(''.center(80, '='))
+ print('|%s|' % ''.center(78, ' '))
+ print('|%s|' % 'VirtualCoach Integration Test Script'.center(78, ' '))
+ print('|%s|' % ''.center(78, ' '))
+ print('|%s|' % ''.center(78, ' '))
if args.oidc_username:
- print '|%s|' % ('OIDC is enabled, user: %s' % args.oidc_username).center(78, ' ')
- print '|%s|' % 'Note: you may be prompted for a password.'.center(78, ' ')
+ print('|%s|' % ('OIDC is enabled, user: %s' % args.oidc_username).center(78, ' '))
+ print('|%s|' % 'Note: you may be prompted for a password.'.center(78, ' '))
elif args.storage_username:
- print '|%s|' % ('Local Storage is enabled, user: %s' % args.storage_username).\
- center(78, ' ')
- print '|%s|' % 'Note: you may be prompted for a password.'.center(78, ' ')
+ print('|%s|' % ('Local Storage is enabled, user: %s' % args.storage_username)
+ .center(78, ' '))
+ print('|%s|' % 'Note: you may be prompted for a password.'.center(78, ' '))
else:
raise ValueError('|%s|' % 'No OIDC or local Storage credentials were provided. Please run '
'the script with either oidc_username or storage_username as arguments'.
center(78, ' '))
- print '|%s|' % ''.center(78, ' ')
- print '=%s=' % ''.center(78, '=')
+ print('|%s|' % ''.center(78, ' '))
+ print('=%s=' % ''.center(78, '='))
# run all of the sequential tests, results are returned upon success or first failure
- print
- print 'Running Test Cases, This May Take A Few Moments...'.center(80, ' ')
- print
+ print()
+ print('Running Test Cases, This May Take A Few Moments...'.center(80, ' '))
+ print()
cumulative = TestCase('Summary'.center(52, ' '))
results = run(args.oidc_username, args.storage_username)
cumulative.done(all([r.success for r in results]))
@@ -540,9 +546,9 @@ if __name__ == '__main__':
# so we just do it manually here instead, not the most readable but functional
widths = [75, 12, 12]
headers = ['Description', 'Duration', 'Result']
- print '\n\n+{0}+{1}+{2}+'.format(*(''.center(w, '-') for w in widths))
- print '|{0}|{1}|{2}|'.format(*(h.center(w, ' ') for h, w in zip(headers, widths)))
- print '+{0}+{1}+{2}+'.format(*(''.center(w, '=') for w in widths))
+ print('\n\n+{0}+{1}+{2}+'.format(*(''.center(w, '-') for w in widths)))
+ print('|{0}|{1}|{2}|'.format(*(h.center(w, ' ') for h, w in zip(headers, widths))))
+ print('+{0}+{1}+{2}+'.format(*(''.center(w, '=') for w in widths)))
# print all of the data rows in easy to visually scan colors (since no one will look at the
# individual test cases and output otherwise
@@ -563,8 +569,8 @@ if __name__ == '__main__':
formatted[2].center(widths[2] + 9, ' ')]
# print the row and next table separator
- print '|{0}|{1}|{2}|'.format(*formatted)
- print '+{0}+{1}+{2}+'.format(*(''.center(w, '-') for w in widths))
+ print('|{0}|{1}|{2}|'.format(*formatted))
+ print('+{0}+{1}+{2}+'.format(*(''.center(w, '-') for w in widths)))
# change the exit code so it can be checked automatically if needed
sys.exit(0 if results[-1].success else -1)
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
index 118d9a7ed40e9438494a80e640f244f5a9726189..268adb598cb95d73edf8934da1c38311df8620a0 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/config.py
@@ -25,12 +25,17 @@
A strongly validated configuration implementation for the VirtualCoach.
"""
-from hbp_nrp_virtual_coach.version import VERSION
+# pylint: disable=W0622
+
+from builtins import str
import json
import logging
import os
+from hbp_nrp_virtual_coach.version import VERSION
+
+
logger = logging.getLogger('Configuration')
@@ -61,7 +66,7 @@ class Config(dict):
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.json')
if not os.path.isfile(path):
raise IOError('[config] config.json not found, terminating.')
- elif not os.access(path, os.R_OK):
+ if not os.access(path, os.R_OK):
raise IOError('[config] config.json is not readable, terminating.')
# parse the config.json and store all values in this object
@@ -95,7 +100,7 @@ class Config(dict):
# convenience, prepend the proxy url to all proxy services, we cannot do this
# for the simulation services because they are backend/experiment id specific
- for k, v in self['proxy-services'].iteritems():
+ for k, v in self['proxy-services'].items():
self['proxy-services'][k] = '%s/%s' % (self['proxy'][environment], v)
def __validate(self, key, values):
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/http_client.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/http_client.py
index e37ce2fe9560d50025eced7605d030d1000de1e8..d6d1e4814ffa8a96a96b5497e4582ef9b101b608 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/http_client.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/http_client.py
@@ -3,6 +3,10 @@ Base HTTP Client class used so that no matter what http client is used
the interface to do the request will always be the same
"""
+# pylint: disable=W0622
+
+from builtins import object
+
class HTTPClient(object):
""" Base HTTP Client class """
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/oidc_http_client.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/oidc_http_client.py
index 4539d77c1403b75c4e84ef267196cd9838df20d7..c8f9570758b34573cd95e93dfd650dcad380f686 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/oidc_http_client.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/oidc_http_client.py
@@ -1,7 +1,7 @@
""" Class which uses the BBP oidc client to do http calls """
-from hbp_nrp_virtual_coach.http_client import HTTPClient
import json
+from hbp_nrp_virtual_coach.http_client import HTTPClient
class OIDCHTTPClient(HTTPClient):
@@ -41,14 +41,11 @@ class OIDCHTTPClient(HTTPClient):
:return: the status of the post request and the content
:rtype: integer, string
"""
- if (type(body) == dict):
- response, content = self.__oidc_client.request(
- url, method='POST', body=json.dumps(body), headers=self.__headers
- )
- else:
- response, content = self.__oidc_client.request(
- url, method='POST', body=body, headers=self.__headers
- )
+ body_ = body if isinstance(body, dict) else json.dumps(body)
+
+ response, content = self.__oidc_client.request(
+ url, method='POST', body=body_, headers=self.__headers
+ )
return int(response['status']), content
def put(self, url, body):
@@ -58,14 +55,11 @@ class OIDCHTTPClient(HTTPClient):
:return: the status of the put request and the content
:rtype: integer, string
"""
- if (type(body) == dict):
- response, content = self.__oidc_client.request(
- url, method='PUT', body=json.dumps(body), headers=self.__headers
- )
- else:
- response, content = self.__oidc_client.request(
- url, method='PUT', body=body, headers=self.__headers
- )
+ body_ = body if isinstance(body, dict) else json.dumps(body)
+
+ response, content = self.__oidc_client.request(
+ url, method='PUT', body=body_, headers=self.__headers
+ )
return int(response['status']), content
def delete(self, url, body):
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/requests_client.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/requests_client.py
index 90084942fa2164e98bf4d260ed252e15b7a73c44..a5ee3089d61318ccd4fee30e821279ffc0817baf 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/requests_client.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/requests_client.py
@@ -1,7 +1,7 @@
""" Class which uses requests to do http calls """
-from hbp_nrp_virtual_coach.http_client import HTTPClient
import requests
+from hbp_nrp_virtual_coach.http_client import HTTPClient
class RequestsClient(HTTPClient):
@@ -25,7 +25,11 @@ class RequestsClient(HTTPClient):
:param url: The url to do a post to
:param body: The content to post to the url
"""
- if (type(body) != dict):
+ try:
+ typeB = isinstance(body)
+ except TypeError:
+ typeB = type(body)
+ if typeB != dict:
response = requests.post(url, headers=self.__headers, data=body)
else:
response = requests.post(url, headers=self.__headers, json=body)
@@ -36,7 +40,11 @@ class RequestsClient(HTTPClient):
:param url: The url to do a request to
:param body: The content to put to
"""
- if (type(body) != dict):
+ try:
+ typeB = isinstance(body)
+ except TypeError:
+ typeB = type(body)
+ if typeB != dict:
response = requests.put(url, headers=self.__headers, data=body)
else:
response = requests.put(url, headers=self.__headers, json=body)
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
index 6bba85b4cc461ac431c44dcd6cb7ba94a538d01a..e9043bab997d31640f77a82e27f5699ded5ff802 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/simulation.py
@@ -25,15 +25,31 @@
An interface to launch or control a simulation instance.
"""
-from hbp_nrp_virtual_coach.config import Config
-from hbp_nrp_virtual_coach.http_client import HTTPClient
+# pylint: disable=W0622
+
+
+from __future__ import print_function
+
+from builtins import str
+from builtins import object
-import httplib
-import json
-import logging
-from urllib2 import HTTPError
import traceback
import os
+import http.client
+import json
+import logging
+
+from six import string_types
+from future import standard_library
+
+from hbp_nrp_virtual_coach.http_client import HTTPClient
+from hbp_nrp_virtual_coach.config import Config
+try:
+ from urllib2 import HTTPError
+except ImportError:
+ from urllib.error import HTTPError # pylint: disable=F0401
+
+standard_library.install_aliases()
class Simulation(object):
@@ -84,10 +100,10 @@ class Simulation(object):
:param server: A string representing the name of the server to try to launch on.
:param reservation: A string representing a cluster resource reservation (if any).
"""
- assert isinstance(experiment_id, str)
- assert isinstance(experiment_conf, str)
- assert isinstance(server, str)
- assert isinstance(reservation, (str, type(None)))
+ assert isinstance(experiment_id, string_types)
+ assert isinstance(experiment_conf, string_types)
+ assert isinstance(server, string_types)
+ assert isinstance(reservation, (string_types, type(None)))
# do not allow reuse of this instance if a simulation has been launched
if self.__server:
@@ -123,11 +139,11 @@ class Simulation(object):
# check to see if the launch was successful, any other failure return codes
# such as 404 will trigger an exception by the OIDCClient itself
- if status_code == httplib.CONFLICT:
+ if status_code == http.client.CONFLICT:
raise Exception(
'Simulation server is launching another experiment.')
- elif status_code != httplib.CREATED:
+ if status_code != http.client.CREATED:
raise Exception(
"Simulation responded with HTTP status %s" % status_code)
@@ -237,7 +253,7 @@ class Simulation(object):
ExDBackend for valid states.
"""
- assert isinstance(state, str)
+ assert isinstance(state, string_types)
# ensure the simulation is started and valid
if not self.__server or not self.__sim_url:
@@ -251,7 +267,7 @@ class Simulation(object):
status_code, _ = self.__http_client.put(url, body={'state': state})
# check the return code, this will return OK if the REST call succeeds
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to set simulation state, HTTP status %s" % status_code)
self.__logger.info('Simulation state: %s', state)
@@ -283,7 +299,7 @@ class Simulation(object):
return
# a loading or shutdown message from the simulation, log it to console, but don't propagate
- elif 'progress' in status:
+ if 'progress' in status:
# ignore duplicate done messages that are used to clear progress bars on the frontend
if 'subtask' not in status['progress']:
@@ -335,7 +351,7 @@ class Simulation(object):
self.__config['simulation-services']['state'])
status_code, content = self.__http_client.get(url)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception("Unable to get current simulation state, HTTP status %s"
% status_code)
@@ -349,7 +365,7 @@ class Simulation(object):
:param script_type: The script type to be retrieved. Either transfer-functions,
state-machines or the brain
"""
- assert isinstance(script_type, (str, unicode))
+ assert isinstance(string_types, string_types)
if not self.__server or not self.__sim_url:
raise Exception(
@@ -359,14 +375,14 @@ class Simulation(object):
raise ValueError("Script type %s not defined. Possible values are: %s"
% (script_type, ", ".join(self.__config['simulation-scripts'])))
- self.__logger.info("Attempting to retrieve %s" % script_type)
+ self.__logger.info("Attempting to retrieve %s", script_type)
url = '%s/%s' % (self.__sim_url,
self.__config['simulation-scripts'][script_type])
status_code, content = self.__http_client.get(url)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception("Unable to get simulation %s, HTTP status %s"
% (script_type, status_code))
return json.loads(content)
@@ -379,11 +395,11 @@ class Simulation(object):
:param script_type: The type of the script to be retrieved (transfer-function, brain,
state-machine)
"""
- assert isinstance(script_name, str)
+ assert isinstance(script_name, string_types)
script_type_print_format = ' '.join(script_type.split('-')).title()
- self.__logger.info('Attempting to retrieve %s: %s' % (script_type_print_format,
- script_name))
+ self.__logger.info('Attempting to retrieve %s: %s', script_type_print_format,
+ script_name)
if not self.__server or not self.__sim_url:
raise Exception("Simulation has not been created, cannot get %s!"
@@ -393,7 +409,7 @@ class Simulation(object):
if script_name not in defined_scripts:
raise ValueError('%s: "%s" does not exist. Please check the %ss ids: \n%s'
% (script_type_print_format, script_name, script_type_print_format,
- str('\n').join(defined_scripts.keys())))
+ str('\n').join(list(defined_scripts.keys()))))
return defined_scripts[script_name]
@@ -410,8 +426,8 @@ class Simulation(object):
:param script: A string containing the code of the new script.
:param new: A flag indicating whether the script is being newly added or modified.
"""
- assert isinstance(script_name, str)
- assert isinstance(script, (str, unicode))
+ assert isinstance(script_name, string_types)
+ assert isinstance(script, string_types)
script_type_display = ' '.join(script_type.split('-')).title()
@@ -421,8 +437,7 @@ class Simulation(object):
defined_scripts = self.__get_simulation_scripts(script_type)['data']
- self.__logger.info('Attempting to set %s %s' %
- (script_type_display, script_name))
+ self.__logger.info('Attempting to set %s %s', script_type_display, script_name)
url = '%s/%s/%s' % (self.__sim_url, self.__config['simulation-scripts'][script_type],
script_name)
started = False
@@ -451,19 +466,18 @@ class Simulation(object):
url = '%s/%s/%s' % (self.__sim_url,
self.__config['simulation-scripts'][script_type], script_name)
status_code, _ = self.__http_client.put(url, body=script)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception("Unable to set %s, HTTP status %s"
% (script_type_display, status_code))
- self.__logger.info("%s '%s' successfully updated" %
- (script_type_display, script_name))
+ self.__logger.info("%s '%s' successfully updated", script_type_display, script_name)
except HTTPError as err:
self.__logger.info(err)
if not new:
self.__logger.info(
- 'Attempting to restore the old %s.' % script_type_display)
+ 'Attempting to restore the old %s.', script_type_display)
status_code, _ = self.__http_client.put(
url, body=script_original)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception("Unable to restore %s, HTTP status %s"
% (script_type_display, status_code))
@@ -480,7 +494,7 @@ class Simulation(object):
:param script_type: A string containing the type of the script to be deleted.
:param script_name: A string containing the name of the script to be deleted.
"""
- assert isinstance(script_name, str)
+ assert isinstance(script_name, string_types)
script_type_display = ' '.join(script_type.split('-')).title()
@@ -495,31 +509,37 @@ class Simulation(object):
% (script_type_display, script_name, script_type_display,
str('\n'.join(script_list))))
- self.__logger.info('Attempting to delete %s %s' %
- (script_type_display, script_name))
+ self.__logger.info('Attempting to delete %s %s', script_type_display, script_name)
url = '%s/%s/%s' % (self.__sim_url, self.__config['simulation-scripts'][script_type],
script_name)
status_code = self.__http_client.delete(url, body=script_name)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception("Unable to delete %s, HTTP status %s"
% (script_type_display, status_code))
- self.__logger.info("%s '%s' successfully deleted" %
- (script_type_display, script_name))
+ self.__logger.info("%s '%s' successfully deleted", script_type_display, script_name)
def print_transfer_functions(self):
"""
Prints a list of the transfer-function names defined in the experiment.
"""
- print "\n".join(self.__get_simulation_scripts(
- 'transfer-function')['data'].keys())
+ printStr = '\n'.join(list(self.__get_simulation_scripts(
+ 'transfer-function')['data'].keys()))
+ try:
+ print(unicode(printStr))
+ except NameError:
+ print(printStr)
def print_state_machines(self):
"""
Prints a list of the state-machine names defined in the experiment.
"""
- print "\n".join(self.__get_simulation_scripts(
- 'state-machine')['data'].keys())
+ printStr = "\n".join(list(self.__get_simulation_scripts(
+ 'state-machine')['data'].keys()))
+ try:
+ print(unicode(printStr))
+ except NameError:
+ print(printStr)
def get_transfer_function(self, transfer_function_name):
"""
@@ -575,15 +595,15 @@ class Simulation(object):
self.__config['simulation-scripts'][experiment_data_type])
else:
url = '%s/%s/%s' % (self.__config['proxy-services']['save-data'],
- self.__experiment_id,
- self.__config['proxy-save'][experiment_data_type])
+ self.__experiment_id,
+ self.__config['proxy-save'][experiment_data_type])
try:
method_fun = getattr(self.__http_client, method)
status_code, _ = method_fun(url, body=experiment_data)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception('Error status code %s' % status_code)
- self.__logger.info("Saved %s." % experiment_data_type)
+ self.__logger.info("Saved %s.", experiment_data_type)
except Exception as err:
self.__logger.info(err)
raise Exception("Failed to save %s." % experiment_data_type)
@@ -592,8 +612,8 @@ class Simulation(object):
"""
Saves the current transfer functions to the storage
"""
- transfer_functions = self.__get_simulation_scripts(
- 'transfer-function')['data'].keys()
+ transfer_functions = list(self.__get_simulation_scripts(
+ 'transfer-function')['data'].keys())
transfer_functions_list = []
for tf in transfer_functions:
transfer_functions_list.append(self.get_transfer_function(str(tf)))
@@ -607,7 +627,7 @@ class Simulation(object):
Saves the current state machines to the storage
"""
state_machines_dict = {}
- for sm in self.__get_simulation_scripts('state-machine')['data'].keys():
+ for sm in list(self.__get_simulation_scripts('state-machine')['data'].keys()):
state_machines_dict[sm] = self.get_state_machine(str(sm))
self.__save_experiment_data('state-machine', {'experiment': self.__experiment_id,
@@ -623,7 +643,7 @@ class Simulation(object):
# Remove the regex entry from the populations dictionary if it is available. The regex
# entry is provided by the frontend to check for duplicate population names and should be
# refactored.
- if 'regex' in populations.values()[0]:
+ if 'regex' in list(populations.values())[0]:
for pop in populations:
del populations[pop]['regex']
@@ -714,7 +734,7 @@ class Simulation(object):
started = True
try:
status_code, _ = self.__http_client.put(url, body=body)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to set Populations, HTTP status %s" % status_code)
self.__logger.info("Populations successfully updated.")
@@ -726,7 +746,7 @@ class Simulation(object):
body['brain_type'] = old_brain_type
status_code, _ = self.__http_client.put(url, body=body)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to restore Brain, HTTP status %s" % status_code)
@@ -746,7 +766,7 @@ class Simulation(object):
integers, lists of integers or python slices.
"""
- assert isinstance(brain_script, (str, unicode))
+ assert isinstance(brain_script, (string_types, string_types))
assert isinstance(neural_population, dict)
if not self.__server or not self.__sim_url:
@@ -776,7 +796,7 @@ class Simulation(object):
# check for current simulation state and pause simulation if it's running
try:
status_code, _ = self.__http_client.put(url, body=body)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to set Brain, HTTP status %s" % status_code)
self.__logger.info("Brain successfully updated.")
@@ -786,7 +806,7 @@ class Simulation(object):
body['data'] = old_brain
body['populations'] = old_populations
status_code, _ = self.__http_client.put(url, body=body)
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to restore Brain, HTTP status %s" % status_code)
@@ -842,27 +862,27 @@ class Simulation(object):
config file.
"""
- assert isinstance(reset_type, (str, unicode))
+ assert isinstance(reset_type, (string_types, string_types))
# NRRPLT-7855
if reset_type in ['full', 'world']:
raise ValueError('Reset %s temporarily disabled due to known Gazebo issue' % reset_type)
- if reset_type not in self.__config['reset-services'].keys():
+ if reset_type not in self.__config['reset-services']:
raise ValueError('Undefined reset type. Possible values are: %s'
- % ", ".join(self.__config['reset-services'].keys()))
+ % ", ".join(list(self.__config['reset-services'].keys())))
# pausing simulation before attempting to reset
self.pause()
- self.__logger.info("Attempting to reset %s" % reset_type)
+ self.__logger.info("Attempting to reset %s", reset_type)
url = '%s/%s/%s' % (self.__sim_url, self.__experiment_id,
self.__config['simulation-services']['reset'])
body = {'resetType': self.__config['reset-services'][reset_type]}
status_code, _ = self.__http_client.put(url, body=body)
# check the return code, this will return OK if the REST call succeeds
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
self.start()
raise Exception(
"Unable to reset simulation, HTTP status %s" % status_code)
@@ -885,7 +905,7 @@ class Simulation(object):
"""
- assert isinstance(cmd, str)
+ assert isinstance(cmd, string_types)
# ensure the simulation is started and valid
if not self.__server or not self.__sim_url:
@@ -901,7 +921,7 @@ class Simulation(object):
status_code, result = self.__http_client.post(url, body=cmd)
# check the return code, this will return OK if the REST call succeeds
- if status_code != httplib.OK:
+ if status_code != http.client.OK:
raise Exception(
"Unable to send command to recorder, HTTP status %s" % status_code)
@@ -932,9 +952,9 @@ class Simulation(object):
filename = os.path.splitext(filename)[0]
filename = filename + '.txt'
self.__vc.set_experiment_file(self.__experiment_id,
- os.path.join('recordings',
- filename),
- description)
+ os.path.join('recordings',
+ filename),
+ description)
self.__send_recorder_cmd('reset')
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_config.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_config.py
index 6f13d2116d8d12457ef37d1e407bcd50cc91a463..2bb14bdc7d30addafee2903c6a778c2c5e7c65d1 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_config.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_config.py
@@ -64,7 +64,7 @@ class TestConfig(unittest.TestCase):
def test_update_proxy_services(self):
config = Config('local')
- for k, v in config['proxy-services'].iteritems():
+ for k, v in config['proxy-services'].items():
self.assertEqual(v, '%s/%s' % (self._conf['proxy']['local'], self._conf['proxy-services'][k]))
def test_environment_from_version(self):
@@ -73,7 +73,7 @@ class TestConfig(unittest.TestCase):
environment = 'dev' if 'dev' in VERSION else 'staging'
config = Config(None)
- for k, v in config['proxy-services'].iteritems():
+ for k, v in config['proxy-services'].items():
self.assertEqual(v, '%s/%s' % (self._conf['proxy'][environment], self._conf['proxy-services'][k]))
@patch('hbp_nrp_virtual_coach.config.json.load')
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
index a69ff870915c85a8d6a836205e203612707ffe32..e4747101fe511194939d29e1bb763af04f1c0aee 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_simulation.py
@@ -25,6 +25,9 @@
Unit tests for the Virtual Coach simulation interface.
"""
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
from hbp_nrp_virtual_coach.simulation import Simulation
from hbp_nrp_virtual_coach.requests_client import RequestsClient
@@ -33,10 +36,10 @@ from hbp_nrp_virtual_coach.config import Config
from mock import Mock, patch, call
import unittest
-import httplib
-from urllib2 import HTTPError
+import http.client
+from urllib.error import HTTPError
import json
-from StringIO import StringIO
+from io import StringIO
class TestSimulation(unittest.TestCase):
@@ -48,7 +51,7 @@ class TestSimulation(unittest.TestCase):
get_response = (None,
'{"serverJobLocation": "mock-location",' \
'"gzweb": {"nrp-services": "mock-services"}}')
- post_response = (httplib.CREATED, '{"simulationID": "12"}')
+ post_response = (http.client.CREATED, '{"simulationID": "12"}')
self._sim._Simulation__http_client.get = Mock(return_value=get_response)
self._sim._Simulation__http_client.post = Mock(return_value=post_response)
@@ -78,7 +81,7 @@ class TestSimulation(unittest.TestCase):
@patch('hbp_nrp_virtual_coach.simulation.traceback')
def test_failed_create_conflict(self, mocked_traceback):
self.setUpForLaunch()
- self._sim._Simulation__http_client.post = Mock(return_value=(httplib.CONFLICT, '{}'))
+ self._sim._Simulation__http_client.post = Mock(return_value=(http.client.CONFLICT, '{}'))
self.assertEqual(self._sim.launch('id', 'conf', 'server-name', None), False)
self._sim._Simulation__http_client.post.assert_called_once()
@@ -87,7 +90,7 @@ class TestSimulation(unittest.TestCase):
@patch('hbp_nrp_virtual_coach.simulation.traceback')
def test_failed_create_other(self, mocked_traceback):
self.setUpForLaunch()
- self._sim._Simulation__http_client.post = Mock(return_value=(httplib.NOT_FOUND, '{}'))
+ self._sim._Simulation__http_client.post = Mock(return_value=(http.client.NOT_FOUND, '{}'))
self.assertEqual(self._sim.launch('id', 'conf', 'server-name', None), False)
self._sim._Simulation__http_client.post.assert_called_once()
@@ -127,15 +130,25 @@ class TestSimulation(unittest.TestCase):
if name == 'rospy' or name == 'std_msgs.msg' or name == 'cle_ros_msgs.msg':
return mock_rospy
return real_import(name, *args)
-
- with patch('__builtin__.__import__', side_effect=mock_import):
- self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
- mock_rospy.Subscriber.assert_has_calls([call('/ros_cle_simulation/status',
- mock_rospy.String,
- self._sim._Simulation__on_status),
- call('/ros_cle_simulation/cle_error',
- mock_rospy.CLEError,
- self._sim._Simulation__on_error)])
+
+ try:
+ with patch('__builtin__.__import__', side_effect=mock_import):
+ self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
+ mock_rospy.Subscriber.assert_has_calls([call('/ros_cle_simulation/status',
+ mock_rospy.String,
+ self._sim._Simulation__on_status),
+ call('/ros_cle_simulation/cle_error',
+ mock_rospy.CLEError,
+ self._sim._Simulation__on_error)])
+ except ModuleNotFoundError:
+ with patch('builtins.__import__', side_effect=mock_import):
+ self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
+ mock_rospy.Subscriber.assert_has_calls([call('/ros_cle_simulation/status',
+ mock_rospy.String,
+ self._sim._Simulation__on_status),
+ call('/ros_cle_simulation/cle_error',
+ mock_rospy.CLEError,
+ self._sim._Simulation__on_error)])
def test_create_without_rospy(self):
@@ -159,9 +172,14 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__logger = Mock()
- with patch('__builtin__.__import__', side_effect=mock_import_fail):
- self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
- mock_rospy.assert_not_called()
+ try:
+ with patch('__builtin__.__import__', side_effect=mock_import_fail):
+ self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
+ mock_rospy.assert_not_called()
+ except ModuleNotFoundError:
+ with patch('builtins.__import__', side_effect=mock_import_fail):
+ self.assertEqual(self._sim.launch('id', 'conf', 'server-name', 'reservation'), True)
+ mock_rospy.assert_not_called()
def test_set_state_asserts(self):
self.assertRaises(AssertionError, self._sim._Simulation__set_state, None)
@@ -172,7 +190,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__sim_url = 'url'
# mock the HTTP call
- self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK, None))
+ self._sim._Simulation__http_client.put = Mock(return_value=(http.client.OK, None))
self._sim._Simulation__set_state('initialized')
self._sim._Simulation__http_client.put.assert_called_once()
@@ -182,7 +200,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__sim_url = 'url'
# mock the HTTP call
- self._sim._Simulation__http_client.put = Mock(return_value=(httplib.NOT_FOUND, None))
+ self._sim._Simulation__http_client.put = Mock(return_value=(http.client.NOT_FOUND, None))
self.assertRaises(Exception, self._sim._Simulation__set_state, 'started')
self._sim._Simulation__http_client.put.assert_called_once()
@@ -215,7 +233,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__sim_url = 'url'
# mock the OIDC call
- self._sim._Simulation__http_client.get = Mock(return_value=(httplib.OK,
+ self._sim._Simulation__http_client.get = Mock(return_value=(http.client.OK,
'{"state": "{}"}'))
self._sim.get_state()
@@ -228,7 +246,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__sim_url = 'url'
# mock the OIDC call
- self._sim._Simulation__http_client.get = Mock(return_value=(httplib.NOT_FOUND,
+ self._sim._Simulation__http_client.get = Mock(return_value=(http.client.NOT_FOUND,
None))
self.assertRaises(Exception, self._sim.get_state)
self._sim._Simulation__http_client.get.assert_called_once()
@@ -248,7 +266,7 @@ class TestSimulation(unittest.TestCase):
# mock the oidc call, the get_all_transfer_function call, the start call, the pause call
# and the get_state call
- self._sim._Simulation__http_client.get = Mock(return_value=(httplib.NOT_FOUND,
+ self._sim._Simulation__http_client.get = Mock(return_value=(http.client.NOT_FOUND,
None))
self.assertRaises(Exception, self._sim._Simulation__get_simulation_scripts, 'foo')
@@ -277,16 +295,16 @@ class TestSimulation(unittest.TestCase):
self.assertRaises(Exception, self._sim.add_transfer_function, 'foo')
- http_error = HTTPError("url", 404, "message", {}, file)
+ http_error = HTTPError("url", 404, "message", {}, None)
self._sim._Simulation__http_client.put.side_effect = [http_error,
- (httplib.NOT_FOUND,
+ (http.client.NOT_FOUND,
None)]
self.assertRaises(Exception, self._sim.edit_state_machine, 'foo', 'import os\n')
self._sim._Simulation__http_client.put.side_effect = [http_error,
- (httplib.OK, None)]
+ (http.client.OK, None)]
self.assertRaises(Exception, self._sim.add_state_machine, 'bar', 'import os\n')
def test_get_transfer_function_asserts(self):
@@ -332,7 +350,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__get_simulation_scripts.return_value = {'brain_type': 'py',
'data_type': 'text'}
self._sim._Simulation__http_client.put = Mock()
- self._sim._Simulation__http_client.put.return_value = (httplib.OK,
+ self._sim._Simulation__http_client.put.return_value = (http.client.OK,
'{"data": {"foo": ""}}')
self._sim.get_state = Mock()
self._sim.get_state.return_value = 'started'
@@ -352,15 +370,15 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__http_client.put.return_value = (HTTPError, None)
self.assertRaises(Exception, self._sim.edit_brain, 'import os\n')
- http_error = HTTPError("url", 404, "message", {}, file)
+ http_error = HTTPError("url", 404, "message", {}, None)
self._sim._Simulation__http_client.put.side_effect = [http_error,
- (httplib.NOT_FOUND,
+ (http.client.NOT_FOUND,
None)]
self.assertRaises(Exception, self._sim.edit_brain, 'foo')
self._sim._Simulation__http_client.put.side_effect = [http_error,
- (httplib.OK, None)]
+ (http.client.OK, None)]
self.assertRaises(Exception, self._sim.edit_brain, 'foo')
def test_edit_populations(self):
@@ -376,7 +394,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__get_simulation_scripts = Mock(return_value={'data_type': 'foo',
'brain_type': 'foo',
'brain_populations': 'bar'})
- self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK,
+ self._sim._Simulation__http_client.put = Mock(return_value=(http.client.OK,
{}))
self._sim.edit_populations({'foo': 'bar'})
@@ -401,9 +419,9 @@ class TestSimulation(unittest.TestCase):
# mock the http call, the get_all_transfer_function call, the start call, the pause call
# and the get_state call
- self._sim._Simulation__http_client.get = Mock(return_value=(httplib.OK,
+ self._sim._Simulation__http_client.get = Mock(return_value=(http.client.OK,
'{"data": {"foo": ""}}'))
- self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK,None))
+ self._sim._Simulation__http_client.put = Mock(return_value=(http.client.OK,None))
self._sim._Simulation__get_all_transfer_functions = Mock()
self._sim._Simulation__get_all_transfer_functions.return_value = {'foo': ''}
@@ -432,7 +450,7 @@ class TestSimulation(unittest.TestCase):
# mock the http call, the get_all_transfer_function call, the start call, the pause call
# and the get_state call
- self._sim._Simulation__http_client.delete = Mock(return_value=httplib.OK)
+ self._sim._Simulation__http_client.delete = Mock(return_value=http.client.OK)
self._sim._Simulation__get_simulation_scripts = Mock()
self._sim._Simulation__get_simulation_scripts.return_value = {'data': {'foo': '', 'bar': ''}}
@@ -445,7 +463,7 @@ class TestSimulation(unittest.TestCase):
self.assertRaises(ValueError, self._sim.delete_state_machine, 'nonExistentScript')
- self._sim._Simulation__http_client.delete.return_value = (httplib.NOT_FOUND,
+ self._sim._Simulation__http_client.delete.return_value = (http.client.NOT_FOUND,
None)
self.assertRaises(Exception, self._sim.delete_state_machine, 'foo')
@@ -475,7 +493,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__headers = {}
- self._sim._Simulation__http_client.put = Mock(return_value=(httplib.OK, None))
+ self._sim._Simulation__http_client.put = Mock(return_value=(http.client.OK, None))
self._sim.save_transfer_functions()
self._sim._Simulation__http_client.put.assert_called_once_with(
@@ -504,7 +522,7 @@ class TestSimulation(unittest.TestCase):
'%s/%s/brain' % (proxyurl, exp_id),
body={'populations': populations,'brain': 'some brain code'})
- self._sim._Simulation__http_client.post = Mock(return_value=(httplib.OK, None))
+ self._sim._Simulation__http_client.post = Mock(return_value=(http.client.OK, None))
self._sim.save_world()
self._sim._Simulation__http_client.post.assert_called_once_with(
'%s/sdf_world' % (self._sim._Simulation__sim_url),
@@ -521,7 +539,18 @@ class TestSimulation(unittest.TestCase):
self._sim.print_transfer_functions()
self._sim.print_state_machines()
- self.assertEqual(mock_stdout.getvalue().strip(), 'foobar\nfoo\nbar\nfoobar\nfoo\nbar')
+ # The order in which 'foobar', 'foo' and 'bar' appear is no longer guaranteed in python 3.3+
+ printStr = mock_stdout.getvalue().strip()
+ self.assertEqual(len(printStr), 29)
+ n = printStr.count('\n')
+ self.assertEqual(n, 5)
+ n = printStr.count('foobar')
+ self.assertEqual(n, 2)
+ n = printStr.count('foo')
+ self.assertEqual(n, 4)
+ n = printStr.count('bar')
+ self.assertEqual(n, 4)
+ #self.assertEqual(mock_stdout.getvalue().strip(), 'foobar\nfoo\nbar\nfoobar\nfoo\nbar')
self._sim._Simulation__get_simulation_scripts.assert_called_twice()
def test_register_status_callback(self):
@@ -621,14 +650,14 @@ class TestSimulation(unittest.TestCase):
self._sim.pause = Mock()
self._sim._Simulation__http_client.put = Mock()
- self._sim._Simulation__http_client.put.return_value = (httplib.OK, None)
+ self._sim._Simulation__http_client.put.return_value = (http.client.OK, None)
# NRRPLT-7855
self.assertRaises(ValueError, self._sim.reset, 'world')
# self._sim.reset('world')
# self._sim._Simulation__http_client.put.assert_called_once()
- self._sim._Simulation__http_client.put.return_value = (httplib.NOT_FOUND,
+ self._sim._Simulation__http_client.put.return_value = (http.client.NOT_FOUND,
None)
self._sim.start = Mock()
@@ -653,7 +682,7 @@ class TestSimulation(unittest.TestCase):
self._sim._Simulation__sim_url = 'url'
self._sim._Simulation__vc = Mock()
self._sim._Simulation__http_client.post = Mock()
- self._sim._Simulation__http_client.post.return_value = (httplib.OK, '{"filename":"name.txt"}')
+ self._sim._Simulation__http_client.post.return_value = (http.client.OK, '{"filename":"name.txt"}')
self._sim.start_recording()
self._sim._Simulation__http_client.post.assert_called_with(u'url/recorder/start', body='start')
self._sim.pause_recording()
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
index 5a98e3a1f98fa89440af298d1245c78cd12c2c6e..48fa333e0af15d1d4a9b20ab489d81199b81cb96 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/tests/test_virtual_coach.py
@@ -25,9 +25,28 @@
Unit tests for the Virtual Coach main interface.
"""
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
from hbp_nrp_virtual_coach.virtual_coach import VirtualCoach
-from bbp_client.oidc.client import BBPOIDCClient
+try:
+ from bbp_client.oidc.client import BBPOIDCClient
+except ImportError:
+ # Currently, bbp_client is still Python2
+ import urllib.parse, urllib.error, sys, os, builtins
+ from bbp_client.oidc.openidconnect import error
+ from bbp_client.oidc.oauth2client import clientsecrets
+ # Python3 only allows absolute paths. In order to allow bbp_client to import
+ # using relative paths, the following two folders have to be added to the import path:
+ sys.path.append(os.path.dirname(error.__file__))
+ sys.path.append(os.path.dirname(clientsecrets.__file__))
+ # Rename the following Python2-packages such that they refer to their Python3 counterparts:
+ sys.modules['urlparse'] = urllib.parse
+ sys.modules['urllib2'] = urllib.error
+ sys.modules['__builtin__'] = builtins
+ from bbp_client.oidc.client import BBPOIDCClient
+
from mock import Mock, patch, MagicMock
import unittest
@@ -38,7 +57,7 @@ import logging
import copy
from dateutil import parser
import json
-from StringIO import StringIO
+from io import StringIO
import os
@@ -56,7 +75,11 @@ class TestVirtualCoach(unittest.TestCase):
def rospy_import_fail(name, globals=globals(), locals=locals(), fromlist=[], level=-1):
if name == 'rospy':
raise ImportError('no ROS for tests')
- return realimport(name, globals, locals, fromlist, level)
+ try:
+ return realimport(name, globals, locals, fromlist, level)
+ except:
+ pass
+ return realimport(name, globals, locals, fromlist, 0)
builtins.__import__ = rospy_import_fail
self._tests_path = os.path.dirname(os.path.realpath(__file__))
diff --git a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
index b5a5e896e258fd9485c85fc58409ae2c20644823..b59b32bf2439bfc854cbc9e899db12ddc5ad32ba 100644
--- a/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
+++ b/hbp_nrp_virtual_coach/hbp_nrp_virtual_coach/virtual_coach.py
@@ -25,25 +25,42 @@
Virtual Coach main entry point for interacting with the experiment list and launching simulations.
"""
-from hbp_nrp_virtual_coach.config import Config
-from hbp_nrp_virtual_coach.simulation import Simulation
-from hbp_nrp_virtual_coach.requests_client import RequestsClient
-from hbp_nrp_virtual_coach.oidc_http_client import OIDCHTTPClient
+# pylint: disable=W0622
+
+from __future__ import print_function
+
+from builtins import str
+from builtins import object
-from datetime import datetime, timedelta
-from collections import defaultdict
-from dateutil import parser, tz
import json
-import requests
import logging
import getpass
-import httplib
+import http.client
import os
import zipfile
import tempfile
-from texttable import Texttable
+
from copy import copy
+from datetime import datetime, timedelta
+from collections import defaultdict
+from dateutil import parser, tz
+
+import requests
+
+from texttable import Texttable
+
+from six import string_types
+
+from future import standard_library
+
+from hbp_nrp_virtual_coach.config import Config
+from hbp_nrp_virtual_coach.simulation import Simulation
+from hbp_nrp_virtual_coach.requests_client import RequestsClient
+from hbp_nrp_virtual_coach.oidc_http_client import OIDCHTTPClient
+
+standard_library.install_aliases()
+
logger_format = '%(levelname)s: [%(asctime)s - %(name)s] %(message)s'
logging.basicConfig(format=logger_format, level=logging.INFO)
logger = logging.getLogger('VirtualCoach')
@@ -82,10 +99,10 @@ class VirtualCoach(object):
:param storage_password: (optional) A string representing the Storage Server password. If
not supplied, the user will be prompted to enter a password.
"""
- assert isinstance(environment, (str, type(None)))
- assert isinstance(oidc_username, (str, type(None)))
- assert isinstance(oidc_token, (str, type(None)))
- assert isinstance(storage_username, (str, type(None)))
+ assert isinstance(environment, (string_types, type(None)))
+ assert isinstance(oidc_username, (string_types, type(None)))
+ assert isinstance(oidc_token, (string_types, type(None)))
+ assert isinstance(storage_username, (string_types, type(None)))
# ROS node and logger configuration only if rospy is available
# pylint: disable=import-error
@@ -124,7 +141,7 @@ class VirtualCoach(object):
# Set self.__http_headers: it is also used
# as an argument of requests.post() and requests.get()
self.__http_headers = {'Content-Type': 'application/json',
- 'Authorization': authorization}
+ 'Authorization': authorization}
self.__http_client.set_headers(self.__http_headers)
# if a Storage Server username is provided, attempt to login
elif storage_username:
@@ -138,7 +155,7 @@ class VirtualCoach(object):
storage_username, storage_password
)
self.__http_headers = {'Content-Type': 'application/json',
- 'Authorization': authorization}
+ 'Authorization': authorization}
self.__http_client = RequestsClient(self.__http_headers)
else:
raise Exception('Virtual Coach instantiated without storage server credentials or oidc'
@@ -165,7 +182,7 @@ class VirtualCoach(object):
# construct the table of experiments with only minimal useful information
table = Texttable()
table.header(['Configuration', 'Name', 'Configuration Path', 'Description'])
- for name, v in sorted(exp_list.iteritems(), key=lambda x: x[1]['configuration']['name']):
+ for name, v in sorted(iter(exp_list.items()), key=lambda x: x[1]['configuration']['name']):
if v['configuration']['maturity'] != 'production' and not dev:
continue
if dev:
@@ -176,7 +193,7 @@ class VirtualCoach(object):
# display the table
logger.info('List of production%s experiments:', '' if not dev else ' and development')
- print table.draw()
+ print(table.draw())
def print_running_experiments(self, cloned=False):
"""
@@ -189,7 +206,7 @@ class VirtualCoach(object):
# construct a table with minimal useful information
table = Texttable()
table.header(['Configuration', 'Owner', 'Time', 'Timeout', 'State', 'Server'])
- for name, v in sorted(exp_list.iteritems(), key=lambda x: x[1]['configuration']['name']):
+ for name, v in sorted(iter(exp_list.items()), key=lambda x: x[1]['configuration']['name']):
# for any running experiments of this type
for server in v['joinableServers']:
@@ -219,7 +236,7 @@ class VirtualCoach(object):
# display the table
logger.info('All running experiments:')
- print table.draw()
+ print(table.draw())
def print_available_servers(self):
"""
@@ -232,12 +249,16 @@ class VirtualCoach(object):
servers = [server['id'] for server in available_servers]
# add a display value if there are no available servers
- if len(servers) == 0:
+ if not servers:
servers = ['No available servers.']
# print the list of available servers
logger.info('Available servers:')
- print '\n'.join(servers)
+ printStr = '\n'.join(servers)
+ try:
+ print(unicode(printStr))
+ except NameError:
+ print(printStr)
return servers
def launch_experiment(self, experiment_id, server=None, reservation=None, cloned=True):
@@ -255,9 +276,9 @@ class VirtualCoach(object):
:param cloned: (optional) True or False depending on if the launched
is a cloned experiment or not.
"""
- assert isinstance(experiment_id, str)
- assert isinstance(server, (str, type(None)))
- assert isinstance(reservation, (str, type(None)))
+ assert isinstance(experiment_id, string_types)
+ assert isinstance(server, (string_types, type(None)))
+ assert isinstance(reservation, (string_types, type(None)))
# retrieve the list of cloned experiments to verify that the given id is valid for the
# backend
@@ -286,15 +307,15 @@ class VirtualCoach(object):
servers = [server]
# if there are no available servers, abort
- if len(servers) == 0:
+ if not servers:
raise ValueError('No available servers for %s, try again later.' % experiment_id)
# attempt to launch the simulation on all server targets, on success return an interface
# to the simulation
sim = Simulation(self.__http_client, self.__config, self)
- for server in servers:
+ for server_i in servers:
try:
- if sim.launch(experiment_id, str(experiment_conf), str(server),
+ if sim.launch(experiment_id, str(experiment_conf), str(server_i),
reservation, cloned):
return sim
@@ -314,11 +335,11 @@ class VirtualCoach(object):
:param exp_id: The id of the experiment to be cloned
:returns: The ID of the cloned experiment
"""
- assert isinstance(exp_id, str)
+ assert isinstance(exp_id, string_types)
exp = self.__get_experiment_list()
- if exp_id not in exp.keys():
+ if exp_id not in list(exp.keys()):
raise ValueError('Experiment ID: "%s" is invalid, please check the list of all '
- 'experiments: \n%s' % (exp_id, '\n'.join(exp.keys())))
+ 'experiments: \n%s' % (exp_id, '\n'.join(list(exp.keys()))))
exp_config_path = exp[exp_id]['configuration']['experimentConfiguration']
body = {'expPath': exp_config_path}
@@ -326,9 +347,8 @@ class VirtualCoach(object):
self.__config['proxy-services']['experiment-clone'], body=body)
if status_code != 200:
raise Exception('Cloning Experiment failed, Status Code: %s' % status_code)
- else:
- logger.info('Experiment "%s" cloned successfully', exp_id)
- return content
+ logger.info('Experiment "%s" cloned successfully', exp_id)
+ return content
def delete_cloned_experiment(self, exp_id):
"""
@@ -353,7 +373,7 @@ class VirtualCoach(object):
:returns: A dict containing the ID of the cloned experiment and the ID of the original
experiment. Dict Keys are: 'clonedExp' and 'originalExp'
"""
- assert isinstance(experiment_id, str)
+ assert isinstance(experiment_id, string_types)
exp_list = self.__get_experiment_list(cloned=True)
if experiment_id not in exp_list:
@@ -373,9 +393,8 @@ class VirtualCoach(object):
headers=self.__http_headers)
if res.status_code != 200:
raise Exception('Cloning Experiment failed, Status Code: %s' % res.status_code)
- else:
- logger.info('Experiment "%s" cloned successfully', experiment_id)
- return res.content
+ logger.info('Experiment "%s" cloned successfully', experiment_id)
+ return res.content
def print_cloned_experiments(self):
"""
@@ -387,7 +406,7 @@ class VirtualCoach(object):
table.header(['Name'])
for experiment in exp_list:
table.add_row([experiment])
- print table.draw()
+ print(table.draw())
def __get_storage_token(self, user_name, password):
"""
@@ -396,16 +415,15 @@ class VirtualCoach(object):
:param user_name: string representing the Storage Server username
:param password: string representing the Storage Server password
"""
- assert isinstance(user_name, str)
- assert isinstance(password, str)
+ assert isinstance(user_name, string_types)
+ assert isinstance(password, string_types)
response = requests.post(self.__config['proxy-services']['storage-authentication'],
json={'user': user_name, 'password': password})
if response.status_code != 200:
raise Exception('Storage Server authentication failed, Status Code: %d'
% response.status_code)
- else:
- return response.content
+ return response.content
def __get_experiment_list(self, cloned=False):
"""
@@ -422,10 +440,9 @@ class VirtualCoach(object):
# return a simple list containing only experiment names since this is the only
# information in the dictionary anyway
return {experiment['name']: experiment for experiment in json.loads(response.content)}
- else:
- _, response = self.__http_client.get(
- self.__config['proxy-services']['experiment-list'])
- return json.loads(response)
+ _, response = self.__http_client.get(
+ self.__config['proxy-services']['experiment-list'])
+ return json.loads(response)
def __get_available_server_list(self):
"""
@@ -435,7 +452,7 @@ class VirtualCoach(object):
status_code, response = self.__http_client.get(
self.__config['proxy-services']['available-servers'])
- if str(status_code) != str(httplib.OK):
+ if str(status_code) != str(http.client.OK):
raise Exception('Error when getting server list, Status Code: %d. Error: %s'
% (status_code, response))
return json.loads(response)
@@ -449,7 +466,7 @@ class VirtualCoach(object):
response = requests.get(self.__config['proxy-services']['csv-files'] % (experiment_id,),
headers=self.__http_headers)
- if response.status_code != httplib.OK:
+ if response.status_code != http.client.OK:
raise Exception('Error when getting CSV files Status Code: %d. Error: %s'
% (response.status_code, response))
csv_files = json.loads(response.content)
@@ -471,11 +488,11 @@ class VirtualCoach(object):
table.set_cols_align(['r', 'c', 'r'])
for i, run_date in enumerate(sorted(csv_files.keys())):
- run_size = sum(file['size'] for file in csv_files[run_date].values())
+ run_size = sum(file['size'] for file in list(csv_files[run_date].values()))
table.add_row([i, run_date, run_size])
logger.info('List of simulation runs')
- print table.draw()
+ print(table.draw())
def print_run_csv_files(self, exp_id, run_id):
"""
@@ -495,11 +512,11 @@ class VirtualCoach(object):
raise Exception('Could not find run %i, %i runs were found' %
(run_id, len(sorted_runs)))
- for csv_file in csv_files[sorted_runs[run_id]].values():
+ for csv_file in list(csv_files[sorted_runs[run_id]].values()):
table.add_row([csv_file['name'], csv_file['size']])
logger.info('Run %i list of files.', run_id)
- print table.draw()
+ print(table.draw())
def print_last_run_csv_files(self, exp_id):
"""
@@ -517,11 +534,11 @@ class VirtualCoach(object):
if not sorted_runs:
raise Exception('Could not find any run')
- for csv_file in csv_files[sorted_runs[-1]].values():
+ for csv_file in list(csv_files[sorted_runs[-1]].values()):
table.add_row([csv_file['name'], csv_file['size']])
logger.info('Last run list of files')
- print table.draw()
+ print(table.draw())
def __get_csv_file_content(self, exp_id, file_uuid):
"""
@@ -533,10 +550,10 @@ class VirtualCoach(object):
logger.info('Retrieving CSV file.')
response = requests.get(self.__config['proxy-services']['experiment-file'] %
- (exp_id, file_uuid),
+ (exp_id, file_uuid),
headers=self.__http_headers)
- if response.status_code != httplib.OK:
+ if response.status_code != http.client.OK:
raise Exception('Error when getting CSV file Status Code: %d. Error: %s'
% (response.status_code, response))
@@ -557,7 +574,7 @@ class VirtualCoach(object):
(run_id, len(sorted_runs)))
if file_name not in csv_files[sorted_runs[run_id]]:
- file_names = ', '.join(f['name'] for f in csv_files[sorted_runs[run_id]].values())
+ file_names = ', '.join(f['name'] for f in list(csv_files[sorted_runs[run_id]].values()))
raise Exception('Could not find file \'%s\' in run %i, available file names are: %s' %
(file_name, run_id, file_names))
@@ -578,7 +595,8 @@ class VirtualCoach(object):
raise Exception('Could not find any run')
if file_name not in csv_files[sorted_runs[-1]]:
- file_names = ', '.join(file['name'] for file in csv_files[sorted_runs[-1]].values())
+ file_names = ', '.join(
+ file['name'] for file in list(csv_files[sorted_runs[-1]].values()))
raise Exception('Could not find file \'%s\' in last run, available file names are: %s' %
(file_name, file_names))
@@ -601,7 +619,7 @@ class VirtualCoach(object):
file_headers['Content-Type'] = 'text/plain'
response = requests.post(url, data=file_content, headers=file_headers)
- if response.status_code != httplib.OK:
+ if response.status_code != http.client.OK:
raise Exception('Error when setting file: %d. Error: %s'
% (response.status_code, response))
@@ -640,7 +658,7 @@ class VirtualCoach(object):
logger.error('The folder %s could not be zipped', dirpath)
raise e
zip_file.close()
- content = open(temp, 'r').read()
+ content = open(temp, 'rb').read()
os.remove(temp)
return content
@@ -651,7 +669,7 @@ class VirtualCoach(object):
:param path: path to the experiment folder or to the zip file to be imported
:type path: str
"""
- if not isinstance(path, str):
+ if not isinstance(path, string_types):
raise TypeError('The provided argument is not a string.')
if not os.path.isfile(path) and not os.path.isdir(path):
raise ValueError('The file or folder named %(path)s does not exist.' % {'path': path})
diff --git a/hbp_nrp_virtual_coach/requirements.txt b/hbp_nrp_virtual_coach/requirements.txt
index 4da393e62a30f471f3da583d58e6243a31c50da9..2f5d8d543b4584c50951821e012f25224336fa38 100644
--- a/hbp_nrp_virtual_coach/requirements.txt
+++ b/hbp_nrp_virtual_coach/requirements.txt
@@ -2,14 +2,12 @@
bbp-client==0.4.4 # EPFL_SPECIFIC
# third-party dependencies
+future==0.18.2
texttable==0.8.7
progressbar2==3.34.0
httplib2==0.12.1
pandas==0.22.0
-ipywidgets==7.5.1
+ipywidgets<=7.5.1; python_version<"3.0" # v8.0.0 incompatible with python 2
jupyter==1.0.0
requests
ipykernel==4.6.0
-
-# unit test dependencies
-mock==1.0.1
diff --git a/hbp_nrp_virtual_coach/requirements_extension_tests.txt b/hbp_nrp_virtual_coach/requirements_extension_tests.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a8a6bd0ac91e29d306bcbd31604dfc7c1fdb4aaa
--- /dev/null
+++ b/hbp_nrp_virtual_coach/requirements_extension_tests.txt
@@ -0,0 +1,2 @@
+#the following is required for the unit testing
+mock==1.0.1
\ No newline at end of file
diff --git a/hbp_nrp_virtual_coach/setup.py b/hbp_nrp_virtual_coach/setup.py
index 1bab64006a93d94deda77718f2ef5e8dc5ce28bb..0aaba7119a263dc7e5bfeb99981000c1606e9752 100644
--- a/hbp_nrp_virtual_coach/setup.py
+++ b/hbp_nrp_virtual_coach/setup.py
@@ -1,31 +1,30 @@
'''setup.py'''
-# pylint: disable=F0401,E0611,W0142
+# pylint: disable=F0401,E0611,W0622,E0012,W0142,W0402
+from builtins import str
try:
from setuptools import setup
except ImportError:
from distutils.core import setup
-import hbp_nrp_virtual_coach
+from optparse import Option # pylint:disable=deprecated-module
import pip
+import hbp_nrp_virtual_coach
-from optparse import Option
options = Option('--workaround')
options.skip_requirements_regex = None
reqs_file = './requirements.txt'
+
+pip_version_major = int(pip.__version__.split('.')[0])
# Hack for old pip versions
-if pip.__version__.startswith('10.'):
- # Versions greater or equal to 10.x don't rely on pip.req.parse_requirements
- install_reqs = list(val.strip() for val in open(reqs_file))
- reqs = install_reqs
-elif pip.__version__.startswith('1.'):
+if pip_version_major == 1:
# Versions 1.x rely on pip.req.parse_requirements
# but don't require a "session" parameter
from pip.req import parse_requirements # pylint:disable=no-name-in-module, import-error
install_reqs = parse_requirements(reqs_file, options=options)
reqs = [str(ir.req) for ir in install_reqs]
-else:
+elif 10 > pip_version_major > 1:
# Versions greater than 1.x but smaller than 10.x rely on pip.req.parse_requirements
# and requires a "session" parameter
from pip.req import parse_requirements # pylint:disable=no-name-in-module, import-error
@@ -37,7 +36,10 @@ else:
options=options
)
reqs = [str(ir.req) for ir in install_reqs]
-
+elif pip_version_major >= 10:
+ # Versions greater or equal to 10.x don't rely on pip.req.parse_requirements
+ install_reqs = list(val.strip() for val in open(reqs_file))
+ reqs = install_reqs
config = {
'description': 'Virtual Coach command-line Neurorobotics Platform interface for HBP SP10',
'author': 'HBP Neurorobotics',
@@ -49,6 +51,8 @@ config = {
'package_data': {
'hbp_nrp_virtual_coach': ['config.json']
},
+ 'classifiers': ['Programming Language :: Python :: 3',
+ "Programming Language :: Python :: 2.7"],
'scripts': [],
'name': 'hbp-nrp-virtual-coach',
'include_package_data': True,