openstack/swift
Code Issues Proposed changes

Merge "Remove duplicate definition of empty string etag"

Browse Source
This commit is contained in:
Zuul 2024-12-03 18:09:52 +00:00 committed by Gerrit Code Review
commit 6e37329bd6
6 changed files with 53 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
swift/common/middleware/crypto/encrypter.py
View File

@ -166,7 +166,7 @@ class EncInputWrapper(object):
# value, with the crypto parameters appended. We use the
# container key here so that only that key is required to
# decrypt all etag values in a container listing when handling
# a container GET request. Don't encrypt an EMPTY_ETAG
# a container GET request. Don't encrypt an MD5_OF_EMPTY_STRING
# unless there actually was some body content, in which case
# the container-listing etag is possibly conveying some
# non-obvious information.

2
test/unit/__init__.py
View File

@ -68,8 +68,6 @@ import inspect
from unittest import SkipTest
EMPTY_ETAG = md5(usedforsecurity=False).hexdigest()
# try not to import this module from swift
if not os.path.basename(sys.argv[0]).startswith('swift'):
# never patch HASH_PATH_SUFFIX AGAIN!

32
test/unit/common/middleware/crypto/test_encrypter.py
View File

@ -28,10 +28,9 @@ from swift.common.middleware.crypto.crypto_utils import (
from swift.common.swob import (
Request, HTTPException, HTTPCreated, HTTPAccepted, HTTPOk, HTTPBadRequest,
wsgi_to_bytes, bytes_to_wsgi)
from swift.common.utils import FileLikeIter
from swift.common.utils import FileLikeIter, MD5_OF_EMPTY_STRING
from test.debug_logger import debug_logger
from test.unit import EMPTY_ETAG
from test.unit.common.middleware.crypto.crypto_helpers import (
fetch_crypto_keys, md5hex, FAKE_IV, encrypt)
from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts
@ -184,11 +183,11 @@ class TestEncrypter(unittest.TestCase):
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
plaintext_etag = MD5_OF_EMPTY_STRING
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': EMPTY_ETAG,
hdrs = {'etag': MD5_OF_EMPTY_STRING,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
@ -209,7 +208,7 @@ class TestEncrypter(unittest.TestCase):
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
self.assertEqual(MD5_OF_EMPTY_STRING, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
@ -229,7 +228,7 @@ class TestEncrypter(unittest.TestCase):
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(b'', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
self.assertEqual(MD5_OF_EMPTY_STRING, resp.headers['Etag'])
def _test_PUT_with_other_footers(self, override_etag):
# verify handling of another middleware's footer callback
@ -332,9 +331,9 @@ class TestEncrypter(unittest.TestCase):
self._test_PUT_with_other_footers('override etag')
def test_PUT_with_other_footers_and_etag_of_empty_body(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_other_footers(EMPTY_ETAG)
# verify that an override etag value of MD5_OF_EMPTY_STRING will be
# encrypted when there was a non-zero body length
self._test_PUT_with_other_footers(MD5_OF_EMPTY_STRING)
def _test_PUT_with_etag_override_in_headers(self, override_etag):
# verify handling of another middleware's
@ -389,9 +388,9 @@ class TestEncrypter(unittest.TestCase):
self._test_PUT_with_etag_override_in_headers('override_etag')
def test_PUT_with_etag_of_empty_body_override_in_headers(self):
# verify that an override etag value of EMPTY_ETAG will be encrypted
# when there was a non-zero body length
self._test_PUT_with_etag_override_in_headers(EMPTY_ETAG)
# verify that an override etag value of MD5_OF_EMPTY_STRING will be
# encrypted when there was a non-zero body length
self._test_PUT_with_etag_override_in_headers(MD5_OF_EMPTY_STRING)
def _test_PUT_with_empty_etag_override_in_headers(self, plaintext):
# verify that an override etag value of '' from other middleware is
@ -543,7 +542,7 @@ class TestEncrypter(unittest.TestCase):
# check that an upstream footer callback gets called
other_footers = {
'Etag': EMPTY_ETAG,
'Etag': MD5_OF_EMPTY_STRING,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
@ -590,8 +589,9 @@ class TestEncrypter(unittest.TestCase):
# if upstream footer override etag is for an empty body then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'X-Object-Sysmeta-Container-Update-Override-Etag': EMPTY_ETAG}
'Etag': MD5_OF_EMPTY_STRING,
'X-Object-Sysmeta-Container-Update-Override-Etag':
MD5_OF_EMPTY_STRING}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
@ -613,7 +613,7 @@ class TestEncrypter(unittest.TestCase):
# if upstream footer override etag is an empty string then check that
# it is not encrypted
other_footers = {
'Etag': EMPTY_ETAG,
'Etag': MD5_OF_EMPTY_STRING,
'X-Object-Sysmeta-Container-Update-Override-Etag': ''}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})

49
test/unit/container/test_backend.py
View File

@ -42,7 +42,8 @@ from swift.common.db import DatabaseAlreadyExists, GreenDBConnection, \
TombstoneReclaimer, GreenDBCursor
from swift.common.request_helpers import get_reserved_name
from swift.common.utils import Timestamp, encode_timestamps, hash_path, \
ShardRange, make_db_file_path, md5, ShardRangeList, Namespace
ShardRange, make_db_file_path, md5, ShardRangeList, Namespace, \
MD5_OF_EMPTY_STRING
from swift.common.storage_policy import POLICIES
import mock
@ -50,7 +51,7 @@ import mock
from test import annotate_failure
from test.debug_logger import debug_logger
from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
EMPTY_ETAG, mock_timestamp_now)
mock_timestamp_now)
from test.unit.common import test_db
@ -263,8 +264,8 @@ class TestContainerBroker(test_db.TestDbBase):
# adding an object makes us unreclaimable
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker.merge_items([dict(obj)])
self.assertFalse(broker.is_reclaimable(float(next(self.ts)), 0))
# ... and "not deleted"
@ -301,8 +302,8 @@ class TestContainerBroker(test_db.TestDbBase):
def check_object_counted(broker_to_test, broker_with_object):
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker_with_object.merge_items([dict(obj)])
self.assertFalse(broker_to_test.is_deleted())
info, deleted = broker_to_test.get_info_is_deleted()
@ -317,8 +318,8 @@ class TestContainerBroker(test_db.TestDbBase):
def check_object_not_counted(broker):
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker.merge_items([dict(obj)])
self.assertTrue(broker.is_deleted())
info, deleted = broker.get_info_is_deleted()
@ -403,8 +404,8 @@ class TestContainerBroker(test_db.TestDbBase):
def check_object_counted(broker_to_test, broker_with_object):
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker_with_object.merge_items([dict(obj)])
self.assertFalse(broker_to_test.empty())
# and delete it
@ -438,7 +439,7 @@ class TestContainerBroker(test_db.TestDbBase):
self.assertTrue(broker.empty())
broker.put_object('o', next(self.ts).internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
own_sr = broker.get_own_shard_range()
self.assertEqual(0, own_sr.object_count)
broker.merge_shard_ranges([own_sr])
@ -508,8 +509,8 @@ class TestContainerBroker(test_db.TestDbBase):
def check_object_counted(broker_to_test, broker_with_object):
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker_with_object.merge_items([dict(obj)])
self.assertFalse(broker_to_test.empty())
# and delete it
@ -527,7 +528,7 @@ class TestContainerBroker(test_db.TestDbBase):
self.assertTrue(broker.empty())
broker.put_object('o', next(self.ts).internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
own_sr = broker.get_own_shard_range()
self.assertEqual(0, own_sr.object_count)
broker.merge_shard_ranges([own_sr])
@ -589,8 +590,8 @@ class TestContainerBroker(test_db.TestDbBase):
def check_object_counted(broker_to_test, broker_with_object):
obj = {'name': 'o', 'created_at': next(self.ts).internal,
'size': 0, 'content_type': 'text/plain', 'etag': EMPTY_ETAG,
'deleted': 0}
'size': 0, 'content_type': 'text/plain',
'etag': MD5_OF_EMPTY_STRING, 'deleted': 0}
broker_with_object.merge_items([dict(obj)])
self.assertFalse(broker_to_test.empty())
# and delete it
@ -609,7 +610,7 @@ class TestContainerBroker(test_db.TestDbBase):
self.assertTrue(broker.empty())
broker.put_object('o', next(self.ts).internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
own_sr = broker.get_own_shard_range()
self.assertEqual(0, own_sr.object_count)
broker.merge_shard_ranges([own_sr])
@ -2626,11 +2627,11 @@ class TestContainerBroker(test_db.TestDbBase):
@with_tempdir
def test_remove_objects(self, tempdir):
objects = (('undeleted', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG, 0, 0),
MD5_OF_EMPTY_STRING, 0, 0),
('other_policy', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG, 0, 1),
MD5_OF_EMPTY_STRING, 0, 1),
('deleted', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG, 1, 0))
MD5_OF_EMPTY_STRING, 1, 0))
object_names = [o[0] for o in objects]
def get_rows(broker):
@ -2750,12 +2751,12 @@ class TestContainerBroker(test_db.TestDbBase):
timestamps = [next(self.ts) for o in obj_names]
for name, timestamp in zip(obj_names, timestamps):
broker.put_object(name, timestamp.internal,
0, 'text/plain', EMPTY_ETAG)
0, 'text/plain', MD5_OF_EMPTY_STRING)
broker._commit_puts() # ensure predictable row order
timestamps = [next(self.ts) for o in obj_names[10:]]
for name, timestamp in zip(obj_names[10:], timestamps):
broker.put_object(name, timestamp.internal,
0, 'text/plain', EMPTY_ETAG, deleted=1)
0, 'text/plain', MD5_OF_EMPTY_STRING, deleted=1)
broker._commit_puts() # ensure predictable row order
# sanity check
@ -3101,7 +3102,7 @@ class TestContainerBroker(test_db.TestDbBase):
obj_create_params = {
'size': 0,
'content_type': 'application/test',
'etag': EMPTY_ETAG,
'etag': MD5_OF_EMPTY_STRING,
}
failures = []
for expected in expectations:
@ -3608,7 +3609,7 @@ class TestContainerBroker(test_db.TestDbBase):
broker.initialize(next(ts).internal, 1)
broker.put_object('b', next(ts).internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
with mock.patch('swift.container.backend.tpool') as mock_tpool:
broker.get_info()

12
test/unit/container/test_replicator.py
View File

@ -30,14 +30,14 @@ from swift.container import replicator, backend, server, sync_store
from swift.container.reconciler import (
MISPLACED_OBJECTS_ACCOUNT, get_reconciler_container_name)
from swift.common.utils import Timestamp, encode_timestamps, ShardRange, \
get_db_files, make_db_file_path
get_db_files, make_db_file_path, MD5_OF_EMPTY_STRING
from swift.common.storage_policy import POLICIES
from test import annotate_failure
from test.debug_logger import debug_logger
from test.unit.common import test_db_replicator
from test.unit import patch_policies, make_timestamp_iter, mock_check_drive, \
EMPTY_ETAG, attach_fake_replication_rpc, FakeHTTPResponse
attach_fake_replication_rpc, FakeHTTPResponse
from contextlib import contextmanager
@ -1603,7 +1603,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync):
broker_ranges = broker.get_all_shard_range_data()
self.assertShardRangesEqual(shard_ranges, broker_ranges)
broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
# sharding not yet enabled so replication not deferred
daemon = check_replicate(broker_ranges, broker, remote_broker)
self.assertEqual(0, daemon.stats['deferred'])
@ -1717,7 +1717,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync):
remote_broker.initialize(put_time, POLICIES.default.idx)
# put an object into local broker
broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
# get an own shard range into local broker
broker.enable_sharding(Timestamp.now())
self.assertFalse(broker.sharding_initiated())
@ -1798,7 +1798,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync):
remote_broker.initialize(put_time, POLICIES.default.idx)
# put an object into local broker
broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
replicate_hook = mock.MagicMock()
fake_repl_connection = attach_fake_replication_rpc(
@ -1826,7 +1826,7 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync):
remote_broker.enable_sharding(Timestamp.now())
# put an object into local broker
broker.put_object('obj', Timestamp.now().internal, 0, 'text/plain',
EMPTY_ETAG)
MD5_OF_EMPTY_STRING)
replicate_hook = mock.MagicMock()
fake_repl_connection = attach_fake_replication_rpc(

11
test/unit/obj/test_diskfile.py
View File

@ -41,18 +41,17 @@ from gzip import GzipFile
import pyeclib.ec_iface
from eventlet import hubs, timeout, tpool
from swift.obj.diskfile import MD5_OF_EMPTY_STRING, update_auditor_status, \
EUCLEAN
from swift.obj.diskfile import update_auditor_status, EUCLEAN
from test import BaseTestCase
from test.debug_logger import debug_logger
from test.unit import (mock as unit_mock, temptree, mock_check_drive,
patch_policies, EMPTY_ETAG, make_timestamp_iter,
patch_policies, make_timestamp_iter,
DEFAULT_TEST_EC_TYPE, requires_o_tmpfile_support_in_tmp,
encode_frag_archive_bodies, skip_if_no_xattrs)
from swift.obj import diskfile
from swift.common import utils
from swift.common.utils import hash_path, mkdirs, Timestamp, \
encode_timestamps, O_TMPFILE, md5 as _md5
encode_timestamps, O_TMPFILE, md5 as _md5, MD5_OF_EMPTY_STRING
from swift.common import ring
from swift.common.splice import splice
from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \
@ -6414,7 +6413,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value,
'X-Timestamp': ts.internal,
'Content-Length': 0,
'Etag': EMPTY_ETAG,
'Etag': MD5_OF_EMPTY_STRING,
'Content-Type': 'plain/text'}
with df.create() as writer:
try:
@ -6431,7 +6430,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value,
'X-Timestamp': ts.internal,
'Content-Length': 0,
'Etag': EMPTY_ETAG,
'Etag': MD5_OF_EMPTY_STRING,
'Content-Type': 'plain/text'}
with df.create() as writer:
try: