From 9ea5721d516cda16e6472a749a9a6b88d0d2eeec Mon Sep 17 00:00:00 2001 From: Paul Van Eck Date: Tue, 7 Mar 2017 19:10:39 -0800 Subject: [PATCH] Enable py35 functional test env This patch also adds the necessary changes to fix py35 compatibility and ensure that the current functional/unit tests pass. Change-Id: Iaced98e98db29882a35bea1abd98e50dd167b511 --- refstack/api/controllers/products.py | 4 ++-- refstack/api/controllers/validation.py | 2 +- refstack/api/controllers/vendors.py | 3 +-- refstack/api/validators.py | 14 ++++++------ refstack/tests/api/test_profile.py | 4 ++-- refstack/tests/unit/test_api.py | 14 +++++++----- refstack/tests/unit/test_validators.py | 30 ++++++++++++++------------ tox.ini | 14 ++++++++++-- 8 files changed, 50 insertions(+), 35 deletions(-) diff --git a/refstack/api/controllers/products.py b/refstack/api/controllers/products.py index 16e756dd..7e1ca971 100644 --- a/refstack/api/controllers/products.py +++ b/refstack/api/controllers/products.py @@ -176,7 +176,7 @@ class ProductsController(validation.BaseRestControllerWithValidation): if _id not in result: result[_id] = s result[_id]['can_manage'] = True - products = result.values() + products = list(result.values()) except Exception as ex: LOG.exception('An error occurred during ' 'operation with database: %s' % ex) @@ -201,7 +201,7 @@ class ProductsController(validation.BaseRestControllerWithValidation): if not is_admin: admin_only_keys = ['created_by_user', 'created_at', 'updated_at', 'properties'] - for key in product.keys(): + for key in list(product): if key in admin_only_keys: product.pop(key) diff --git a/refstack/api/controllers/validation.py b/refstack/api/controllers/validation.py index 055e7262..6b30861f 100644 --- a/refstack/api/controllers/validation.py +++ b/refstack/api/controllers/validation.py @@ -57,7 +57,7 @@ class BaseRestControllerWithValidation(rest.RestController): def post(self, ): """POST handler.""" self.validator.validate(pecan.request) - item = json.loads(pecan.request.body) + item = json.loads(pecan.request.body.decode('utf-8')) item_id = self.store_item(item) pecan.response.status = 201 return item_id diff --git a/refstack/api/controllers/vendors.py b/refstack/api/controllers/vendors.py index 192a65da..443546dd 100644 --- a/refstack/api/controllers/vendors.py +++ b/refstack/api/controllers/vendors.py @@ -166,12 +166,11 @@ class VendorsController(validation.BaseRestControllerWithValidation): if _id not in result: result[_id] = vendor result[_id]['can_manage'] = True - vendors = result.values() + vendors = list(result.values()) except Exception as ex: LOG.exception('An error occurred during ' 'operation with database: %s' % ex) pecan.abort(400) - return {'vendors': vendors} @pecan.expose('json') diff --git a/refstack/api/validators.py b/refstack/api/validators.py index 50700d38..02602803 100644 --- a/refstack/api/validators.py +++ b/refstack/api/validators.py @@ -65,7 +65,7 @@ class BaseValidator(object): def validate(self, request): """Validate request.""" try: - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) except (ValueError, TypeError) as e: raise api_exc.ValidationError('Malformed request', e) @@ -135,7 +135,7 @@ class TestResultValidator(BaseValidator): raise api_exc.ValidationError('Malformed public key', e) verifier = key.verifier(sign, padding.PKCS1v15(), hashes.SHA256()) - verifier.update(request.body.encode('utf-8')) + verifier.update(request.body) try: verifier.verify() except InvalidSignature: @@ -146,7 +146,7 @@ class TestResultValidator(BaseValidator): def _is_empty_result(self, request): """Check if the test results list is empty.""" - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) if len(body['results']) != 0: return False return True @@ -173,7 +173,7 @@ class PubkeyValidator(BaseValidator): def validate(self, request): """Validate uploaded test results.""" super(PubkeyValidator, self).validate(request) - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) key_format = body['raw_key'].strip().split()[0] if key_format not in ('ssh-dss', 'ssh-rsa', @@ -215,7 +215,7 @@ class VendorValidator(BaseValidator): def validate(self, request): """Validate uploaded vendor data.""" super(VendorValidator, self).validate(request) - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) self.check_emptyness(body, ['name']) @@ -239,7 +239,7 @@ class ProductValidator(BaseValidator): def validate(self, request): """Validate uploaded test results.""" super(ProductValidator, self).validate(request) - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) self.check_emptyness(body, ['name', 'product_type']) @@ -260,6 +260,6 @@ class ProductVersionValidator(BaseValidator): def validate(self, request): """Validate product version data.""" super(ProductVersionValidator, self).validate(request) - body = json.loads(request.body) + body = json.loads(request.body.decode('utf-8')) self.check_emptyness(body, ['version']) diff --git a/refstack/tests/api/test_profile.py b/refstack/tests/api/test_profile.py index 7395396c..35bda032 100644 --- a/refstack/tests/api/test_profile.py +++ b/refstack/tests/api/test_profile.py @@ -62,9 +62,9 @@ class TestProfileEndpoint(api.FunctionalTest): pubkey = key.public_key().public_bytes( serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH - ) + ).decode('utf-8') body = {'raw_key': pubkey, - 'self_signature': binascii.b2a_hex(sign)} + 'self_signature': binascii.b2a_hex(sign).decode('utf-8')} json_params = json.dumps(body) # POST endpoint diff --git a/refstack/tests/unit/test_api.py b/refstack/tests/unit/test_api.py index ecbf4fcb..689147cf 100644 --- a/refstack/tests/unit/test_api.py +++ b/refstack/tests/unit/test_api.py @@ -147,7 +147,7 @@ class ResultsControllerTestCase(BaseControllerTestCase): @mock.patch('refstack.db.store_results') def test_post(self, mock_store_results): - self.mock_request.body = '{"answer": 42}' + self.mock_request.body = b'{"answer": 42}' self.mock_request.headers = {} mock_store_results.return_value = 'fake_test_id' result = self.controller.post() @@ -167,7 +167,7 @@ class ResultsControllerTestCase(BaseControllerTestCase): @mock.patch('refstack.db.get_pubkey') def test_post_with_sign(self, mock_get_pubkey, mock_store_results, mock_get_version, mock_check, mock_foundation): - self.mock_request.body = '{"answer": 42, "cpid": "123"}' + self.mock_request.body = b'{"answer": 42, "cpid": "123"}' self.mock_request.headers = { 'X-Signature': 'fake-sign', 'X-Public-Key': 'ssh-rsa Zm9vIGJhcg==' @@ -438,7 +438,7 @@ class BaseRestControllerWithValidationTestCase(BaseControllerTestCase): @mock.patch('pecan.response') @mock.patch('pecan.request') def test_post(self, mock_request, mock_response): - mock_request.body = '[42]' + mock_request.body = b'[42]' self.controller.store_item = mock.Mock(return_value='fake_id') result = self.controller.post() @@ -724,7 +724,9 @@ class PublicKeysControllerTestCase(BaseControllerTestCase): 'comment': 'Don\'t_Panic.', 'openid': 'fake_id' } - self.mock_request.body = json.dumps({'raw_key': raw_key}) + self.mock_request.body = json.dumps( + {'raw_key': raw_key} + ).encode('utf-8') self.controller.post() self.assertEqual(201, self.mock_response.status) mock_store_pubkey.assert_called_once_with(fake_pubkey) @@ -737,7 +739,9 @@ class PublicKeysControllerTestCase(BaseControllerTestCase): 'comment': '', 'openid': 'fake_id' } - self.mock_request.body = json.dumps({'raw_key': raw_key}) + self.mock_request.body = json.dumps( + {'raw_key': raw_key} + ).encode('utf-8') self.controller.post() mock_store_pubkey.assert_called_once_with(fake_pubkey) diff --git a/refstack/tests/unit/test_validators.py b/refstack/tests/unit/test_validators.py index 57688ada..976b3d1b 100644 --- a/refstack/tests/unit/test_validators.py +++ b/refstack/tests/unit/test_validators.py @@ -97,7 +97,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): def test_validation(self): with mock.patch('jsonschema.validate') as mock_validate: request = mock.Mock() - request.body = json.dumps(self.FAKE_JSON) + request.body = json.dumps(self.FAKE_JSON).encode('utf-8') request.headers = {} self.validator.validate(request) mock_validate.assert_called_once_with(self.FAKE_JSON, @@ -105,7 +105,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): def test_validation_with_signature(self): request = mock.Mock() - request.body = json.dumps(self.FAKE_JSON) + request.body = json.dumps(self.FAKE_JSON).encode('utf-8') key = rsa.generate_private_key( public_exponent=65537, @@ -113,7 +113,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): backend=default_backend() ) signer = key.signer(padding.PKCS1v15(), hashes.SHA256()) - signer.update(request.body.encode('utf-8')) + signer.update(request.body) sign = signer.finalize() pubkey = key.public_key().public_bytes( serialization.Encoding.OpenSSH, @@ -127,7 +127,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): def test_validation_fail_no_json(self): wrong_request = mock.Mock() - wrong_request.body = 'foo' + wrong_request.body = b'foo' self.assertRaises(api_exc.ValidationError, self.validator.validate, wrong_request) @@ -140,7 +140,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): wrong_request = mock.Mock() wrong_request.body = json.dumps({ 'foo': 'bar' - }) + }).encode('utf-8') self.assertRaises(api_exc.ValidationError, self.validator.validate, wrong_request) @@ -151,7 +151,9 @@ class TestResultValidatorTestCase(base.BaseTestCase): def test_validation_fail_with_empty_result(self): wrong_request = mock.Mock() - wrong_request.body = json.dumps(self.FAKE_JSON_WITH_EMPTY_RESULTS) + wrong_request.body = json.dumps( + self.FAKE_JSON_WITH_EMPTY_RESULTS + ).encode('utf-8') self.assertRaises(api_exc.ValidationError, self.validator.validate, wrong_request) @@ -160,7 +162,7 @@ class TestResultValidatorTestCase(base.BaseTestCase): def test_validation_with_broken_signature(self, mock_validate): request = mock.Mock() - request.body = json.dumps(self.FAKE_JSON) + request.body = json.dumps(self.FAKE_JSON).encode('utf-8') key = rsa.generate_private_key( public_exponent=65537, key_size=2048, @@ -224,12 +226,12 @@ class PubkeyValidatorTestCase(base.BaseTestCase): def test_validation(self): request = mock.Mock() - request.body = json.dumps(self.FAKE_JSON) + request.body = json.dumps(self.FAKE_JSON).encode('utf-8') self.validator.validate(request) def test_validation_fail_no_json(self): wrong_request = mock.Mock() - wrong_request.body = 'foo' + wrong_request.body = b'foo' self.assertRaises(api_exc.ValidationError, self.validator.validate, wrong_request) @@ -242,7 +244,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase): wrong_request = mock.Mock() wrong_request.body = json.dumps({ 'foo': 'bar' - }) + }).encode('utf-8') self.assertRaises(api_exc.ValidationError, self.validator.validate, wrong_request) @@ -257,7 +259,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase): body['self_signature'] = 'deadbeef' request = mock.Mock() - request.body = json.dumps(body) + request.body = json.dumps(body).encode('utf-8') try: self.validator.validate(request) except api_exc.ValidationError as e: @@ -269,7 +271,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase): 'self_signature': 'deadbeef' } request = mock.Mock() - request.body = json.dumps(body) + request.body = json.dumps(body).encode('utf-8') try: self.validator.validate(request) except api_exc.ValidationError as e: @@ -281,7 +283,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase): 'self_signature': 'deadbeef?' } request = mock.Mock() - request.body = json.dumps(body) + request.body = json.dumps(body).encode('utf-8') try: self.validator.validate(request) except api_exc.ValidationError as e: @@ -293,7 +295,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase): 'self_signature': 'deadbeef' } request = mock.Mock() - request.body = json.dumps(body) + request.body = json.dumps(body).encode('utf-8') try: self.validator.validate(request) except api_exc.ValidationError as e: diff --git a/tox.ini b/tox.ini index 68181ca1..68dba6bd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] -# py34 tests should be run before py27 +# py3* tests should be run before py27 # it is a workaround for testr bug # https://bugs.launchpad.net/testrepository/+bug/1229445 -envlist = py34,py27,pep8,pip-check-reqs +envlist = py35,py27,pep8,pip-check-reqs minversion = 1.6 skipsdist = True @@ -27,6 +27,16 @@ setenv = SUBUNIT_TEST_PATH=./refstack/tests/api # require cleanup of database commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}' +[testenv:py35-func-mysql] +basepython = python3.5 +setenv = SUBUNIT_TEST_PATH=./refstack/tests/api +# Integration/functional tests +# must not be run in parallel (--concurrency=1), +# because each of these tests +# require cleanup of database +commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}' + + [testenv:pep8] commands = flake8 {posargs}