From a3f2400cbac1fb514a1f5075cfcb7d45b8851d44 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 13 Jan 2014 17:45:56 +0000 Subject: [PATCH] Consolidating and standardizing x-delete-at format Change-Id: Idc916da1c7fe1cc43a2c26f7f7ee1d4fcdd52c89 --- swift/common/utils.py | 23 +++++++++++++++++++ swift/obj/server.py | 15 ++++++------- swift/proxy/controllers/obj.py | 30 ++++++++++++++----------- test/unit/common/test_utils.py | 40 ++++++++++++++++++++++++++++++++++ test/unit/obj/test_server.py | 13 ++++++----- 5 files changed, 95 insertions(+), 26 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 184e1d53a6..fc76be1e6d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -535,6 +535,29 @@ def normalize_timestamp(timestamp): return "%016.05f" % (float(timestamp)) +def normalize_delete_at_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx (10) format. + + Note that timestamps less than 0000000000 are raised to + 0000000000 and values greater than November 20th, 2286 at + 17:46:39 UTC will be capped at that date and time, resulting in + no return value exceeding 9999999999. + + This cap is because the expirer is already working through a + sorted list of strings that were all a length of 10. Adding + another digit would mess up the sort and cause the expirer to + break from processing early. By 2286, this problem will need to + be fixed, probably by creating an additional .expiring_objects + account to work from with 11 (or more) digit container names. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return '%010d' % min(max(0, float(timestamp)), 9999999999) + + def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path diff --git a/swift/obj/server.py b/swift/obj/server.py index 99a5db89cf..5796e8a541 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -29,7 +29,7 @@ from hashlib import md5 from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ - config_true_value, timing_stats, replication + config_true_value, timing_stats, replication, normalize_delete_at_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ check_float, check_utf8 @@ -254,10 +254,7 @@ class ObjectController(object): :param request: the original request driving the update :param objdevice: device name that the object is in """ - # Quick cap that will work from now until Sat Nov 20 17:46:39 2286 - # At that time, Swift will be so popular and pervasive I will have - # created income for thousands of future programmers. - delete_at = max(min(delete_at, 9999999999), 0) + delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None @@ -276,8 +273,8 @@ class ObjectController(object): 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. - delete_at_container = str( - delete_at / self.expiring_objects_container_divisor * + delete_at_container = ( + int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') @@ -300,8 +297,10 @@ class ObjectController(object): # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str( - delete_at / self.expiring_objects_container_divisor * + int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) + delete_at_container = normalize_delete_at_timestamp( + delete_at_container) for host, contdevice in updates: self.async_update( diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 4c27f1446a..840601d7cf 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -41,7 +41,7 @@ from eventlet.timeout import Timeout from swift.common.utils import ContextPool, normalize_timestamp, \ config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ - quorum_size, GreenAsyncPile + quorum_size, GreenAsyncPile, normalize_delete_at_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE @@ -549,7 +549,8 @@ class ObjectController(Controller): return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-After') - req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after) + req.headers['x-delete-at'] = normalize_delete_at_timestamp( + time.time() + x_delete_after) if self.app.object_post_as_copy: req.method = 'PUT' req.path_info = '/v1/%s/%s/%s' % ( @@ -587,8 +588,9 @@ class ObjectController(Controller): return HTTPNotFound(request=req) if 'x-delete-at' in req.headers: try: - x_delete_at = int(req.headers['x-delete-at']) - if x_delete_at < time.time(): + x_delete_at = normalize_delete_at_timestamp( + int(req.headers['x-delete-at'])) + if int(x_delete_at) < time.time(): return HTTPBadRequest( body='X-Delete-At in past', request=req, content_type='text/plain') @@ -597,9 +599,9 @@ class ObjectController(Controller): content_type='text/plain', body='Non-integer X-Delete-At') req.environ.setdefault('swift.log_info', []).append( - 'x-delete-at:%d' % x_delete_at) - delete_at_container = str( - x_delete_at / + 'x-delete-at:%s' % x_delete_at) + delete_at_container = normalize_delete_at_timestamp( + int(x_delete_at) / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) delete_at_part, delete_at_nodes = \ @@ -777,7 +779,8 @@ class ObjectController(Controller): return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-After') - req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after) + req.headers['x-delete-at'] = normalize_delete_at_timestamp( + time.time() + x_delete_after) partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) # do a HEAD request for container sync and checking object versions @@ -928,8 +931,9 @@ class ObjectController(Controller): if 'x-delete-at' in req.headers: try: - x_delete_at = int(req.headers['x-delete-at']) - if x_delete_at < time.time(): + x_delete_at = normalize_delete_at_timestamp( + int(req.headers['x-delete-at'])) + if int(x_delete_at) < time.time(): return HTTPBadRequest( body='X-Delete-At in past', request=req, content_type='text/plain') @@ -937,9 +941,9 @@ class ObjectController(Controller): return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-At') req.environ.setdefault('swift.log_info', []).append( - 'x-delete-at:%d' % x_delete_at) - delete_at_container = str( - x_delete_at / + 'x-delete-at:%s' % x_delete_at) + delete_at_container = normalize_delete_at_timestamp( + int(x_delete_at) / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) delete_at_part, delete_at_nodes = \ diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index b119c4ec1b..db2c9ba3bb 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -210,6 +210,46 @@ class TestUtils(unittest.TestCase): self.assertRaises(ValueError, utils.normalize_timestamp, '') self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') + def test_normalize_delete_at_timestamp(self): + self.assertEquals( + utils.normalize_delete_at_timestamp(1253327593), + '1253327593') + self.assertEquals( + utils.normalize_delete_at_timestamp(1253327593.67890), + '1253327593') + self.assertEquals( + utils.normalize_delete_at_timestamp('1253327593'), + '1253327593') + self.assertEquals( + utils.normalize_delete_at_timestamp('1253327593.67890'), + '1253327593') + self.assertEquals( + utils.normalize_delete_at_timestamp(-1253327593), + '0000000000') + self.assertEquals( + utils.normalize_delete_at_timestamp(-1253327593.67890), + '0000000000') + self.assertEquals( + utils.normalize_delete_at_timestamp('-1253327593'), + '0000000000') + self.assertEquals( + utils.normalize_delete_at_timestamp('-1253327593.67890'), + '0000000000') + self.assertEquals( + utils.normalize_delete_at_timestamp(71253327593), + '9999999999') + self.assertEquals( + utils.normalize_delete_at_timestamp(71253327593.67890), + '9999999999') + self.assertEquals( + utils.normalize_delete_at_timestamp('71253327593'), + '9999999999') + self.assertEquals( + utils.normalize_delete_at_timestamp('71253327593.67890'), + '9999999999') + self.assertRaises(ValueError, utils.normalize_timestamp, '') + self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') + def test_backwards(self): # Test swift.common.utils.backward diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index fd6c11e201..1f8a202ed1 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -2270,8 +2270,8 @@ class TestObjectController(unittest.TestCase): 'DELETE', 2, 'a', 'c', 'o', req, 'sda1') self.assertEquals( given_args, [ - 'DELETE', '.expiring_objects', '0', - '2-a/c/o', None, None, None, + 'DELETE', '.expiring_objects', '0000000000', + '0000000002-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '123', @@ -2296,7 +2296,8 @@ class TestObjectController(unittest.TestCase): self.object_controller.delete_at_update( 'DELETE', -2, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ - 'DELETE', '.expiring_objects', '0', '0-a/c/o', None, None, None, + 'DELETE', '.expiring_objects', '0000000000', '0000000000-a/c/o', + None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '1234', @@ -2352,7 +2353,8 @@ class TestObjectController(unittest.TestCase): req, 'sda1') self.assertEquals( given_args, [ - 'PUT', '.expiring_objects', '0', '2-a/c/o', '127.0.0.1:1234', + 'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o', + '127.0.0.1:1234', '3', 'sdc1', HeaderKeyDict({ 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', @@ -2404,7 +2406,8 @@ class TestObjectController(unittest.TestCase): req, 'sda1') self.assertEquals( given_args, [ - 'DELETE', '.expiring_objects', '0', '2-a/c/o', None, None, + 'DELETE', '.expiring_objects', '0000000000', + '0000000002-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'DELETE http://localhost/v1/a/c/o'}),