From 4ca605c51e44e817ad73e9e7c229d5738b44dd9f Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 5 Oct 2018 14:58:35 -0700 Subject: [PATCH] Move base64-decoding/unpickling into DatabaseBroker ...which was the one responsible for pickling and encoding the data in the first place. Change-Id: Ia7fad7ddd7cf95c8a0168dd6c899c80ddfdd3521 --- swift/account/backend.py | 8 +++----- swift/common/db.py | 5 +++-- swift/container/backend.py | 16 +++++++--------- test/unit/common/test_db.py | 11 +++++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/swift/account/backend.py b/swift/account/backend.py index bd21a697ff..df2954fb70 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -17,7 +17,6 @@ Pluggable Back-end for Account Server """ from uuid import uuid4 -import six.moves.cPickle as pickle import sqlite3 @@ -204,12 +203,11 @@ class AccountBroker(DatabaseBroker): def _commit_puts_load(self, item_list, entry): """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" - loaded = pickle.loads(entry.decode('base64')) # check to see if the update includes policy_index or not (name, put_timestamp, delete_timestamp, object_count, bytes_used, - deleted) = loaded[:6] - if len(loaded) > 6: - storage_policy_index = loaded[6] + deleted) = entry[:6] + if len(entry) > 6: + storage_policy_index = entry[6] else: # legacy support during upgrade until first non legacy storage # policy is defined diff --git a/swift/common/db.py b/swift/common/db.py index 62fb89635a..e504155c6d 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -730,7 +730,8 @@ class DatabaseBroker(object): for entry in fp.read().split(b':'): if entry: try: - self._commit_puts_load(item_list, entry) + data = pickle.loads(base64.b64decode(entry)) + self._commit_puts_load(item_list, data) except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), @@ -760,7 +761,7 @@ class DatabaseBroker(object): def _commit_puts_load(self, item_list, entry): """ - Unmarshall the :param:entry and append it to :param:item_list. + Unmarshall the :param:entry tuple and append it to :param:item_list. This is implemented by a particular broker to be compatible with its :func:`merge_items`. """ diff --git a/swift/container/backend.py b/swift/container/backend.py index 87bb62130e..4a2a8ff1c3 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -21,7 +21,6 @@ import os from uuid import uuid4 import six -import six.moves.cPickle as pickle from six.moves import range import sqlite3 from eventlet import tpool @@ -643,17 +642,16 @@ class ContainerBroker(DatabaseBroker): def _commit_puts_load(self, item_list, entry): """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" - data = pickle.loads(entry.decode('base64')) - (name, timestamp, size, content_type, etag, deleted) = data[:6] - if len(data) > 6: - storage_policy_index = data[6] + (name, timestamp, size, content_type, etag, deleted) = entry[:6] + if len(entry) > 6: + storage_policy_index = entry[6] else: storage_policy_index = 0 content_type_timestamp = meta_timestamp = None - if len(data) > 7: - content_type_timestamp = data[7] - if len(data) > 8: - meta_timestamp = data[8] + if len(entry) > 7: + content_type_timestamp = entry[7] + if len(entry) > 8: + meta_timestamp = entry[8] item_list.append({'name': name, 'created_at': timestamp, 'size': size, diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index effa2d1bea..ebf240e724 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -274,7 +274,7 @@ class ExampleBroker(DatabaseBroker): conn.commit() def _commit_puts_load(self, item_list, entry): - (name, timestamp, deleted) = pickle.loads(base64.b64decode(entry)) + (name, timestamp, deleted) = entry item_list.append({ 'name': name, 'created_at': timestamp, @@ -1422,16 +1422,19 @@ class TestDatabaseBroker(unittest.TestCase): # load file and merge with open(broker.pending_file, 'wb') as fd: - fd.write(b':1:2:99') + for v in (1, 2, 99): + fd.write(b':' + base64.b64encode(pickle.dumps( + v, protocol=PICKLE_PROTOCOL))) with patch.object(broker, 'merge_items') as mock_merge_items: broker._commit_puts_load = lambda l, e: l.append(e) broker._commit_puts() - mock_merge_items.assert_called_once_with([b'1', b'2', b'99']) + mock_merge_items.assert_called_once_with([1, 2, 99]) self.assertEqual(0, os.path.getsize(broker.pending_file)) # load file and merge with given list with open(broker.pending_file, 'wb') as fd: - fd.write(b':bad') + fd.write(b':' + base64.b64encode(pickle.dumps( + b'bad', protocol=PICKLE_PROTOCOL))) with patch.object(broker, 'merge_items') as mock_merge_items: broker._commit_puts_load = lambda l, e: l.append(e) broker._commit_puts([b'not'])