From f05e1e34971a7fb37d4ab183b3c8a69795772a3d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 19 Jun 2015 17:33:29 +0200 Subject: [PATCH] Fix Python 3 issues in glance.tests.unit * api/versions.py: HTTP body is bytes, encode JSON to UTF-8 on Python 3 * urlsafe_encrypt(): replace str with six.binary_type, encode plaintext to UTF-8 if it is Unicode * urlsafe_decrypt(): replace str with six.binary_type, decode result from UTF-8 on Python 3 * MetadefIndex: replace map() with a list comprehension to get a list on Python 3, map() returns an iterator on Python 3 * test_artifact_type_definition_framework: skip sort() test on Python 3 because int and str are not comparable * test_cache_middleware: HTTP body is bytes, replace '' with b'' * test_misc: add checks on the result type of urlsafe_encrypt() and urlsafe_decrypt(). On Python 3, encode plaintext to UTF-8 to compare it to ciphertext. Comparing bytes and str raises a TypeError when python3 is run with the -bb command line option. * tox.ini: add to following glance.tests.unit tests to Python 3.4 - test_artifact_type_definition_framework - test_cache_middleware - test_db_metadef - test_misc - test_policy - test_search - test_versions Change-Id: Id47c5e9ba761a61d1cb80b65b0b6238f4a331c8c --- glance/api/versions.py | 6 ++++- glance/common/crypt.py | 25 +++++++++++++------ glance/search/plugins/metadefs.py | 2 +- ...test_artifact_type_definition_framework.py | 7 +++++- glance/tests/unit/test_cache_middleware.py | 2 +- glance/tests/unit/test_misc.py | 13 ++++++++-- tox.ini | 9 ++++++- 7 files changed, 50 insertions(+), 14 deletions(-) diff --git a/glance/api/versions.py b/glance/api/versions.py index 453d41f740..85f8845888 100644 --- a/glance/api/versions.py +++ b/glance/api/versions.py @@ -15,6 +15,7 @@ from oslo_config import cfg from oslo_serialization import jsonutils +import six from six.moves import http_client import webob.dec @@ -75,7 +76,10 @@ class Controller(object): response = webob.Response(request=req, status=http_client.MULTIPLE_CHOICES, content_type='application/json') - response.body = jsonutils.dumps(dict(versions=version_objs)) + json = jsonutils.dumps(dict(versions=version_objs)) + if six.PY3: + json = json.encode('utf-8') + response.body = json return response @webob.dec.wsgify(RequestClass=wsgi.Request) diff --git a/glance/common/crypt.py b/glance/common/crypt.py index ae5160b0c1..c63a197f5a 100644 --- a/glance/common/crypt.py +++ b/glance/common/crypt.py @@ -24,13 +24,16 @@ import base64 from Crypto.Cipher import AES from Crypto import Random from Crypto.Random import random +import six # NOTE(jokke): simplified transition to py3, behaves like py2 xrange from six.moves import range def urlsafe_encrypt(key, plaintext, blocksize=16): """ - Encrypts plaintext. Resulting ciphertext will contain URL-safe characters + Encrypts plaintext. Resulting ciphertext will contain URL-safe characters. + If plaintext is Unicode, encode it to UTF-8 before encryption. + :param key: AES secret key :param plaintext: Input text to be encrypted :param blocksize: Non-zero integer multiple of AES blocksize in bytes (16) @@ -43,27 +46,35 @@ def urlsafe_encrypt(key, plaintext, blocksize=16): """ pad_length = (blocksize - len(text) % blocksize) sr = random.StrongRandom() - pad = ''.join(chr(sr.randint(1, 0xFF)) for i in range(pad_length - 1)) + pad = b''.join(six.int2byte(sr.randint(1, 0xFF)) + for i in range(pad_length - 1)) # We use chr(0) as a delimiter between text and padding - return text + chr(0) + pad + return text + b'\0' + pad + if isinstance(plaintext, six.text_type): + plaintext = plaintext.encode('utf-8') # random initial 16 bytes for CBC init_vector = Random.get_random_bytes(16) cypher = AES.new(key, AES.MODE_CBC, init_vector) - padded = cypher.encrypt(pad(str(plaintext))) + padded = cypher.encrypt(pad(six.binary_type(plaintext))) return base64.urlsafe_b64encode(init_vector + padded) def urlsafe_decrypt(key, ciphertext): """ - Decrypts URL-safe base64 encoded ciphertext + Decrypts URL-safe base64 encoded ciphertext. + On Python 3, the result is decoded from UTF-8. + :param key: AES secret key :param ciphertext: The encrypted text to decrypt :returns : Resulting plaintext """ # Cast from unicode - ciphertext = base64.urlsafe_b64decode(str(ciphertext)) + ciphertext = base64.urlsafe_b64decode(six.binary_type(ciphertext)) cypher = AES.new(key, AES.MODE_CBC, ciphertext[:16]) padded = cypher.decrypt(ciphertext[16:]) - return padded[:padded.rfind(chr(0))] + text = padded[:padded.rfind(b'\0')] + if six.PY3: + text = text.decode('utf-8') + return text diff --git a/glance/search/plugins/metadefs.py b/glance/search/plugins/metadefs.py index 0b63143218..2ec13d75de 100644 --- a/glance/search/plugins/metadefs.py +++ b/glance/search/plugins/metadefs.py @@ -214,7 +214,7 @@ class MetadefIndex(base.IndexBase): if 'default' in document: document['default'] = str(document['default']) if 'enum' in document: - document['enum'] = map(str, document['enum']) + document['enum'] = [str(enum) for enum in document['enum']] return document diff --git a/glance/tests/unit/test_artifact_type_definition_framework.py b/glance/tests/unit/test_artifact_type_definition_framework.py index a4084e8dd3..e79187285f 100644 --- a/glance/tests/unit/test_artifact_type_definition_framework.py +++ b/glance/tests/unit/test_artifact_type_definition_framework.py @@ -15,6 +15,7 @@ import datetime import mock +import six from glance.common.artifacts import declarative import glance.common.artifacts.definitions as defs @@ -401,7 +402,11 @@ class TestDeclarativeProperties(test_utils.BaseTestCase): self.assertEqual(1234, tt.address[1]) self.assertEqual(True, tt.address[2]) - self.assertRaises(exc.InvalidArtifactPropertyValue, tt.address.sort) + # On Python 3, sort() fails because int (1) and string ("20") are not + # comparable + if six.PY2: + self.assertRaises(exc.InvalidArtifactPropertyValue, + tt.address.sort) self.assertRaises(exc.InvalidArtifactPropertyValue, tt.address.pop, 0) self.assertRaises(exc.InvalidArtifactPropertyValue, tt.address.pop, 1) self.assertRaises(exc.InvalidArtifactPropertyValue, tt.address.pop) diff --git a/glance/tests/unit/test_cache_middleware.py b/glance/tests/unit/test_cache_middleware.py index 57e4c39546..0e0d786cc3 100644 --- a/glance/tests/unit/test_cache_middleware.py +++ b/glance/tests/unit/test_cache_middleware.py @@ -634,7 +634,7 @@ class TestCacheMiddlewareProcessResponse(base.IsolatedUnitTest): resp = webob.Response(request=request) self.assertRaises(webob.exc.HTTPForbidden, cache_filter.process_response, resp) - self.assertEqual([''], resp.app_iter) + self.assertEqual([b''], resp.app_iter) def test_v1_process_response_download_restricted(self): """ diff --git a/glance/tests/unit/test_misc.py b/glance/tests/unit/test_misc.py index 744c00a250..b04e00c61e 100644 --- a/glance/tests/unit/test_misc.py +++ b/glance/tests/unit/test_misc.py @@ -15,6 +15,7 @@ import os +import six # NOTE(jokke): simplified transition to py3, behaves like py2 xrange from six.moves import range @@ -34,13 +35,21 @@ class UtilsTestCase(test_utils.BaseTestCase): plaintext_list = [''] blocksize = 64 for i in range(3 * blocksize): - plaintext_list.append(os.urandom(i)) + text = os.urandom(i) + if six.PY3: + text = text.decode('latin1') + plaintext_list.append(text) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) - self.assertNotEqual(ciphertext, plaintext) + self.assertIsInstance(ciphertext, bytes) + if six.PY3: + self.assertNotEqual(ciphertext, plaintext.encode('utf-8')) + else: + self.assertNotEqual(ciphertext, plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) + self.assertIsInstance(text, str) self.assertEqual(plaintext, text) def test_empty_metadata_headers(self): diff --git a/tox.ini b/tox.ini index 6d8c0e2742..76530f6fca 100644 --- a/tox.ini +++ b/tox.ini @@ -23,23 +23,30 @@ commands = glance.tests.unit.common.test_property_utils \ glance.tests.unit.common.test_scripts \ glance.tests.unit.common.test_swift_store_utils \ + glance.tests.unit.test_artifact_type_definition_framework \ glance.tests.unit.test_artifacts_plugin_loader \ glance.tests.unit.test_auth \ glance.tests.unit.test_cached_images \ + glance.tests.unit.test_cache_middleware \ glance.tests.unit.test_context \ glance.tests.unit.test_context_middleware \ + glance.tests.unit.test_db_metadef \ glance.tests.unit.test_domain \ glance.tests.unit.test_domain_proxy \ glance.tests.unit.test_gateway \ glance.tests.unit.test_image_cache_client \ glance.tests.unit.test_jsonpatchmixin \ glance.tests.unit.test_manage \ + glance.tests.unit.test_misc \ glance.tests.unit.test_notifier \ glance.tests.unit.test_opts \ + glance.tests.unit.test_policy \ glance.tests.unit.test_schema \ glance.tests.unit.test_scrubber \ + glance.tests.unit.test_search \ glance.tests.unit.test_store_artifact \ - glance.tests.unit.test_store_location + glance.tests.unit.test_store_location \ + glance.tests.unit.test_versions [testenv:pep8] commands =