diff --git a/swift/common/middleware/crypto/encrypter.py b/swift/common/middleware/crypto/encrypter.py index c450694b38..d0318d9e89 100644 --- a/swift/common/middleware/crypto/encrypter.py +++ b/swift/common/middleware/crypto/encrypter.py @@ -166,7 +166,7 @@ class EncInputWrapper(object): # value, with the crypto parameters appended. We use the # container key here so that only that key is required to # decrypt all etag values in a container listing when handling - # a container GET request. Don't encrypt an EMPTY_ETAG + # a container GET request. Don't encrypt an MD5_OF_EMPTY_STRING # unless there actually was some body content, in which case # the container-listing etag is possibly conveying some # non-obvious information. diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 18793579d7..484f934eed 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -68,8 +68,6 @@ import inspect from unittest import SkipTest -EMPTY_ETAG = md5(usedforsecurity=False).hexdigest() - # try not to import this module from swift if not os.path.basename(sys.argv[0]).startswith('swift'): # never patch HASH_PATH_SUFFIX AGAIN! diff --git a/test/unit/common/middleware/crypto/test_encrypter.py b/test/unit/common/middleware/crypto/test_encrypter.py index de1c2cf909..5d77cdde89 100644 --- a/test/unit/common/middleware/crypto/test_encrypter.py +++ b/test/unit/common/middleware/crypto/test_encrypter.py @@ -28,10 +28,9 @@ from swift.common.middleware.crypto.crypto_utils import ( from swift.common.swob import ( Request, HTTPException, HTTPCreated, HTTPAccepted, HTTPOk, HTTPBadRequest, wsgi_to_bytes, bytes_to_wsgi) -from swift.common.utils import FileLikeIter +from swift.common.utils import FileLikeIter, MD5_OF_EMPTY_STRING from test.debug_logger import debug_logger -from test.unit import EMPTY_ETAG from test.unit.common.middleware.crypto.crypto_helpers import ( fetch_crypto_keys, md5hex, FAKE_IV, encrypt) from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts @@ -184,11 +183,11 @@ class TestEncrypter(unittest.TestCase): def test_PUT_zero_size_object(self): # object body encryption should be skipped for zero sized object body object_key = fetch_crypto_keys()['object'] - plaintext_etag = EMPTY_ETAG + plaintext_etag = MD5_OF_EMPTY_STRING env = {'REQUEST_METHOD': 'PUT', CRYPTO_KEY_CALLBACK: fetch_crypto_keys} - hdrs = {'etag': EMPTY_ETAG, + hdrs = {'etag': MD5_OF_EMPTY_STRING, 'content-type': 'text/plain', 'content-length': '0', 'x-object-meta-etag': 'not to be confused with the Etag!', @@ -209,7 +208,7 @@ class TestEncrypter(unittest.TestCase): # verify that there is no body crypto meta self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs) # verify etag is md5 of plaintext - self.assertEqual(EMPTY_ETAG, req_hdrs['Etag']) + self.assertEqual(MD5_OF_EMPTY_STRING, req_hdrs['Etag']) # verify there is no etag crypto meta self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs) # verify there is no container update override for etag @@ -229,7 +228,7 @@ class TestEncrypter(unittest.TestCase): get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = get_req.get_response(self.app) self.assertEqual(b'', resp.body) - self.assertEqual(EMPTY_ETAG, resp.headers['Etag']) + self.assertEqual(MD5_OF_EMPTY_STRING, resp.headers['Etag']) def _test_PUT_with_other_footers(self, override_etag): # verify handling of another middleware's footer callback @@ -332,9 +331,9 @@ class TestEncrypter(unittest.TestCase): self._test_PUT_with_other_footers('override etag') def test_PUT_with_other_footers_and_etag_of_empty_body(self): - # verify that an override etag value of EMPTY_ETAG will be encrypted - # when there was a non-zero body length - self._test_PUT_with_other_footers(EMPTY_ETAG) + # verify that an override etag value of MD5_OF_EMPTY_STRING will be + # encrypted when there was a non-zero body length + self._test_PUT_with_other_footers(MD5_OF_EMPTY_STRING) def _test_PUT_with_etag_override_in_headers(self, override_etag): # verify handling of another middleware's @@ -389,9 +388,9 @@ class TestEncrypter(unittest.TestCase): self._test_PUT_with_etag_override_in_headers('override_etag') def test_PUT_with_etag_of_empty_body_override_in_headers(self): - # verify that an override etag value of EMPTY_ETAG will be encrypted - # when there was a non-zero body length - self._test_PUT_with_etag_override_in_headers(EMPTY_ETAG) + # verify that an override etag value of MD5_OF_EMPTY_STRING will be + # encrypted when there was a non-zero body length + self._test_PUT_with_etag_override_in_headers(MD5_OF_EMPTY_STRING) def _test_PUT_with_empty_etag_override_in_headers(self, plaintext): # verify that an override etag value of '' from other middleware is @@ -543,7 +542,7 @@ class TestEncrypter(unittest.TestCase): # check that an upstream footer callback gets called other_footers = { - 'Etag': EMPTY_ETAG, + 'Etag': MD5_OF_EMPTY_STRING, 'X-Object-Sysmeta-Other': 'other sysmeta', 'X-Object-Sysmeta-Container-Update-Override-Etag': 'other override'} @@ -590,8 +589,9 @@ class TestEncrypter(unittest.TestCase): # if upstream footer override etag is for an empty body then check that # it is not encrypted other_footers = { - 'Etag': EMPTY_ETAG, - 'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG} + 'Etag': MD5_OF_EMPTY_STRING, + 'X-Object-Sysmeta-Container-Update-Override-Etag': + MD5_OF_EMPTY_STRING} env.update({'swift.callback.update_footers': lambda footers: footers.update(other_footers)}) req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) @@ -613,7 +613,7 @@ class TestEncrypter(unittest.TestCase): # if upstream footer override etag is an empty string then check that # it is not encrypted other_footers = { - 'Etag': EMPTY_ETAG, + 'Etag': MD5_OF_EMPTY_STRING, 'X-Object-Sysmeta-Container-Update-Override-Etag': ''} env.update({'swift.callback.update_footers': lambda footers: footers.update(other_footers)}) diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index bfb49a3d7c..d378da970e 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -42,7 +42,8 @@ from swift.common.db import DatabaseAlreadyExists, GreenDBConnection, \ TombstoneReclaimer, GreenDBCursor from swift.common.request_helpers import get_reserved_name from swift.common.utils import Timestamp, encode_timestamps, hash_path, \ - ShardRange, make_db_file_path, md5, ShardRangeList, Namespace + ShardRange, make_db_file_path, md5, ShardRangeList, Namespace, \ + MD5_OF_EMPTY_STRING from swift.common.storage_policy import POLICIES import mock @@ -50,7 +51,7 @@ import mock from test import annotate_failure from test.debug_logger import debug_logger from test.unit import (patch_policies, with_tempdir, make_timestamp_iter, - EMPTY_ETAG, mock_timestamp_now) + mock_timestamp_now) from test.unit.common import test_db @@ -263,8 +264,8 @@ class TestContainerBroker(test_db.TestDbBase): # adding an object makes us unreclaimable obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker.merge_items([dict(obj)]) self.assertFalse(broker.is_reclaimable(float(next(self.ts)), 0)) # ... and "not deleted" @@ -301,8 +302,8 @@ class TestContainerBroker(test_db.TestDbBase): def check_object_counted(broker_to_test, broker_with_object): obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker_with_object.merge_items([dict(obj)]) self.assertFalse(broker_to_test.is_deleted()) info, deleted = broker_to_test.get_info_is_deleted() @@ -317,8 +318,8 @@ class TestContainerBroker(test_db.TestDbBase): def check_object_not_counted(broker): obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker.merge_items([dict(obj)]) self.assertTrue(broker.is_deleted()) info, deleted = broker.get_info_is_deleted() @@ -403,8 +404,8 @@ class TestContainerBroker(test_db.TestDbBase): def check_object_counted(broker_to_test, broker_with_object): obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker_with_object.merge_items([dict(obj)]) self.assertFalse(broker_to_test.empty()) # and delete it @@ -438,7 +439,7 @@ class TestContainerBroker(test_db.TestDbBase): self.assertTrue(broker.empty()) broker.put_object('o', next(self.ts).internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) own_sr = broker.get_own_shard_range() self.assertEqual(0, own_sr.object_count) broker.merge_shard_ranges([own_sr]) @@ -508,8 +509,8 @@ class TestContainerBroker(test_db.TestDbBase): def check_object_counted(broker_to_test, broker_with_object): obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker_with_object.merge_items([dict(obj)]) self.assertFalse(broker_to_test.empty()) # and delete it @@ -527,7 +528,7 @@ class TestContainerBroker(test_db.TestDbBase): self.assertTrue(broker.empty()) broker.put_object('o', next(self.ts).internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) own_sr = broker.get_own_shard_range() self.assertEqual(0, own_sr.object_count) broker.merge_shard_ranges([own_sr]) @@ -589,8 +590,8 @@ class TestContainerBroker(test_db.TestDbBase): def check_object_counted(broker_to_test, broker_with_object): obj = {'name': 'o', 'created_at': next(self.ts).internal, - 'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG, - 'deleted': 0} + 'size': 0, 'content_type': 'text/plain', + 'etag': MD5_OF_EMPTY_STRING, 'deleted': 0} broker_with_object.merge_items([dict(obj)]) self.assertFalse(broker_to_test.empty()) # and delete it @@ -609,7 +610,7 @@ class TestContainerBroker(test_db.TestDbBase): self.assertTrue(broker.empty()) broker.put_object('o', next(self.ts).internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) own_sr = broker.get_own_shard_range() self.assertEqual(0, own_sr.object_count) broker.merge_shard_ranges([own_sr]) @@ -2626,11 +2627,11 @@ class TestContainerBroker(test_db.TestDbBase): @with_tempdir def test_remove_objects(self, tempdir): objects = (('undeleted', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG, 0, 0), + MD5_OF_EMPTY_STRING, 0, 0), ('other_policy', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG, 0, 1), + MD5_OF_EMPTY_STRING, 0, 1), ('deleted', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG, 1, 0)) + MD5_OF_EMPTY_STRING, 1, 0)) object_names = [o[0] for o in objects] def get_rows(broker): @@ -2750,12 +2751,12 @@ class TestContainerBroker(test_db.TestDbBase): timestamps = [next(self.ts) for o in obj_names] for name, timestamp in zip(obj_names, timestamps): broker.put_object(name, timestamp.internal, - 0, 'text/plain', EMPTY_ETAG) + 0, 'text/plain', MD5_OF_EMPTY_STRING) broker._commit_puts() # ensure predictable row order timestamps = [next(self.ts) for o in obj_names[10:]] for name, timestamp in zip(obj_names[10:], timestamps): broker.put_object(name, timestamp.internal, - 0, 'text/plain', EMPTY_ETAG, deleted=1) + 0, 'text/plain', MD5_OF_EMPTY_STRING, deleted=1) broker._commit_puts() # ensure predictable row order # sanity check @@ -3101,7 +3102,7 @@ class TestContainerBroker(test_db.TestDbBase): obj_create_params = { 'size': 0, 'content_type': 'application/test', - 'etag': EMPTY_ETAG, + 'etag': MD5_OF_EMPTY_STRING, } failures = [] for expected in expectations: @@ -3608,7 +3609,7 @@ class TestContainerBroker(test_db.TestDbBase): broker.initialize(next(ts).internal, 1) broker.put_object('b', next(ts).internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) with mock.patch('swift.container.backend.tpool') as mock_tpool: broker.get_info() diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index 4c84fcb9dd..ddf863590c 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -30,14 +30,14 @@ from swift.container import replicator, backend, server, sync_store from swift.container.reconciler import ( MISPLACED_OBJECTS_ACCOUNT, get_reconciler_container_name) from swift.common.utils import Timestamp, encode_timestamps, ShardRange, \ - get_db_files, make_db_file_path + get_db_files, make_db_file_path, MD5_OF_EMPTY_STRING from swift.common.storage_policy import POLICIES from test import annotate_failure from test.debug_logger import debug_logger from test.unit.common import test_db_replicator from test.unit import patch_policies, make_timestamp_iter, mock_check_drive, \ - EMPTY_ETAG, attach_fake_replication_rpc, FakeHTTPResponse + attach_fake_replication_rpc, FakeHTTPResponse from contextlib import contextmanager @@ -1603,7 +1603,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): broker_ranges = broker.get_all_shard_range_data() self.assertShardRangesEqual(shard_ranges, broker_ranges) broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) # sharding not yet enabled so replication not deferred daemon = check_replicate(broker_ranges, broker, remote_broker) self.assertEqual(0, daemon.stats['deferred']) @@ -1717,7 +1717,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): remote_broker.initialize(put_time, POLICIES.default.idx) # put an object into local broker broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) # get an own shard range into local broker broker.enable_sharding(Timestamp.now()) self.assertFalse(broker.sharding_initiated()) @@ -1798,7 +1798,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): remote_broker.initialize(put_time, POLICIES.default.idx) # put an object into local broker broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) replicate_hook = mock.MagicMock() fake_repl_connection = attach_fake_replication_rpc( @@ -1826,7 +1826,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): remote_broker.enable_sharding(Timestamp.now()) # put an object into local broker broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain', - EMPTY_ETAG) + MD5_OF_EMPTY_STRING) replicate_hook = mock.MagicMock() fake_repl_connection = attach_fake_replication_rpc( diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 20a8f87ba6..9f6810d8fe 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -41,18 +41,17 @@ from gzip import GzipFile import pyeclib.ec_iface from eventlet import hubs, timeout, tpool -from swift.obj.diskfile import MD5_OF_EMPTY_STRING, update_auditor_status, \ - EUCLEAN +from swift.obj.diskfile import update_auditor_status, EUCLEAN from test import BaseTestCase from test.debug_logger import debug_logger from test.unit import (mock as unit_mock, temptree, mock_check_drive, - patch_policies, EMPTY_ETAG, make_timestamp_iter, + patch_policies, make_timestamp_iter, DEFAULT_TEST_EC_TYPE, requires_o_tmpfile_support_in_tmp, encode_frag_archive_bodies, skip_if_no_xattrs) from swift.obj import diskfile from swift.common import utils from swift.common.utils import hash_path, mkdirs, Timestamp, \ - encode_timestamps, O_TMPFILE, md5 as _md5 + encode_timestamps, O_TMPFILE, md5 as _md5, MD5_OF_EMPTY_STRING from swift.common import ring from swift.common.splice import splice from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \ @@ -6414,7 +6413,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase): meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value, 'X-Timestamp': ts.internal, 'Content-Length': 0, - 'Etag': EMPTY_ETAG, + 'Etag': MD5_OF_EMPTY_STRING, 'Content-Type': 'plain/text'} with df.create() as writer: try: @@ -6431,7 +6430,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase): meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value, 'X-Timestamp': ts.internal, 'Content-Length': 0, - 'Etag': EMPTY_ETAG, + 'Etag': MD5_OF_EMPTY_STRING, 'Content-Type': 'plain/text'} with df.create() as writer: try: