Merge "Consolidating and standardizing x-delete-at format"

swift/common/utils.py

@@ -535,6 +535,29 @@ def normalize_timestamp(timestamp):
     return "%016.05f" % (float(timestamp))
 
 
+def normalize_delete_at_timestamp(timestamp):
+    """
+    Format a timestamp (string or numeric) into a standardized
+    xxxxxxxxxx (10) format.
+
+    Note that timestamps less than 0000000000 are raised to
+    0000000000 and values greater than November 20th, 2286 at
+    17:46:39 UTC will be capped at that date and time, resulting in
+    no return value exceeding 9999999999.
+
+    This cap is because the expirer is already working through a
+    sorted list of strings that were all a length of 10. Adding
+    another digit would mess up the sort and cause the expirer to
+    break from processing early. By 2286, this problem will need to
+    be fixed, probably by creating an additional .expiring_objects
+    account to work from with 11 (or more) digit container names.
+
+    :param timestamp: unix timestamp
+    :returns: normalized timestamp as a string
+    """
+    return '%010d' % min(max(0, float(timestamp)), 9999999999)
+
+
 def mkdirs(path):
     """
     Ensures the path is a directory or makes it if not. Errors if the path

swift/obj/server.py

@@ -29,7 +29,7 @@ from hashlib import md5
 from eventlet import sleep, Timeout
 
 from swift.common.utils import public, get_logger, \
-    config_true_value, timing_stats, replication
+    config_true_value, timing_stats, replication, normalize_delete_at_timestamp
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
     check_float, check_utf8
@@ -254,10 +254,7 @@ class ObjectController(object):
         :param request: the original request driving the update
         :param objdevice: device name that the object is in
         """
-        # Quick cap that will work from now until Sat Nov 20 17:46:39 2286
+        delete_at = normalize_delete_at_timestamp(delete_at)
-        # At that time, Swift will be so popular and pervasive I will have
-        # created income for thousands of future programmers.
-        delete_at = max(min(delete_at, 9999999999), 0)
         updates = [(None, None)]
 
         partition = None
@@ -276,8 +273,8 @@ class ObjectController(object):
                     'best guess as to the container name for now.' % op)
                 # TODO(gholt): In a future release, change the above warning to
                 # a raised exception and remove the guess code below.
-                delete_at_container = str(
+                delete_at_container = (
-                    delete_at / self.expiring_objects_container_divisor *
+                    int(delete_at) / self.expiring_objects_container_divisor *
                     self.expiring_objects_container_divisor)
             partition = headers_in.get('X-Delete-At-Partition', None)
             hosts = headers_in.get('X-Delete-At-Host', '')
@@ -300,8 +297,10 @@ class ObjectController(object):
             # it will be ignored when the expirer eventually tries to issue the
             # object DELETE later since the X-Delete-At value won't match up.
             delete_at_container = str(
-                delete_at / self.expiring_objects_container_divisor *
+                int(delete_at) / self.expiring_objects_container_divisor *
                 self.expiring_objects_container_divisor)
+        delete_at_container = normalize_delete_at_timestamp(
+            delete_at_container)
 
         for host, contdevice in updates:
             self.async_update(

swift/proxy/controllers/obj.py

@@ -41,7 +41,7 @@ from eventlet.timeout import Timeout
 
 from swift.common.utils import ContextPool, normalize_timestamp, \
     config_true_value, public, json, csv_append, GreenthreadSafeIterator, \
-    quorum_size, GreenAsyncPile
+    quorum_size, GreenAsyncPile, normalize_delete_at_timestamp
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_metadata, check_object_creation, \
     CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE
@@ -549,7 +549,8 @@ class ObjectController(Controller):
                 return HTTPBadRequest(request=req,
                                       content_type='text/plain',
                                       body='Non-integer X-Delete-After')
-            req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
+            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
+                time.time() + x_delete_after)
         if self.app.object_post_as_copy:
             req.method = 'PUT'
             req.path_info = '/v1/%s/%s/%s' % (
@@ -587,8 +588,9 @@ class ObjectController(Controller):
                 return HTTPNotFound(request=req)
             if 'x-delete-at' in req.headers:
                 try:
-                    x_delete_at = int(req.headers['x-delete-at'])
+                    x_delete_at = normalize_delete_at_timestamp(
-                    if x_delete_at < time.time():
+                        int(req.headers['x-delete-at']))
+                    if int(x_delete_at) < time.time():
                         return HTTPBadRequest(
                             body='X-Delete-At in past', request=req,
                             content_type='text/plain')
@@ -597,9 +599,9 @@ class ObjectController(Controller):
                                           content_type='text/plain',
                                           body='Non-integer X-Delete-At')
                 req.environ.setdefault('swift.log_info', []).append(
-                    'x-delete-at:%d' % x_delete_at)
+                    'x-delete-at:%s' % x_delete_at)
-                delete_at_container = str(
+                delete_at_container = normalize_delete_at_timestamp(
-                    x_delete_at /
+                    int(x_delete_at) /
                     self.app.expiring_objects_container_divisor *
                     self.app.expiring_objects_container_divisor)
                 delete_at_part, delete_at_nodes = \
@@ -777,7 +779,8 @@ class ObjectController(Controller):
                 return HTTPBadRequest(request=req,
                                       content_type='text/plain',
                                       body='Non-integer X-Delete-After')
-            req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
+            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
+                time.time() + x_delete_after)
         partition, nodes = self.app.object_ring.get_nodes(
             self.account_name, self.container_name, self.object_name)
         # do a HEAD request for container sync and checking object versions
@@ -928,8 +931,9 @@ class ObjectController(Controller):
 
         if 'x-delete-at' in req.headers:
             try:
-                x_delete_at = int(req.headers['x-delete-at'])
+                x_delete_at = normalize_delete_at_timestamp(
-                if x_delete_at < time.time():
+                    int(req.headers['x-delete-at']))
+                if int(x_delete_at) < time.time():
                     return HTTPBadRequest(
                         body='X-Delete-At in past', request=req,
                         content_type='text/plain')
@@ -937,9 +941,9 @@ class ObjectController(Controller):
                 return HTTPBadRequest(request=req, content_type='text/plain',
                                       body='Non-integer X-Delete-At')
             req.environ.setdefault('swift.log_info', []).append(
-                'x-delete-at:%d' % x_delete_at)
+                'x-delete-at:%s' % x_delete_at)
-            delete_at_container = str(
+            delete_at_container = normalize_delete_at_timestamp(
-                x_delete_at /
+                int(x_delete_at) /
                 self.app.expiring_objects_container_divisor *
                 self.app.expiring_objects_container_divisor)
             delete_at_part, delete_at_nodes = \

test/unit/common/test_utils.py

@@ -210,6 +210,46 @@ class TestUtils(unittest.TestCase):
         self.assertRaises(ValueError, utils.normalize_timestamp, '')
         self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
 
+    def test_normalize_delete_at_timestamp(self):
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(1253327593),
+            '1253327593')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(1253327593.67890),
+            '1253327593')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('1253327593'),
+            '1253327593')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('1253327593.67890'),
+            '1253327593')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(-1253327593),
+            '0000000000')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(-1253327593.67890),
+            '0000000000')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('-1253327593'),
+            '0000000000')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('-1253327593.67890'),
+            '0000000000')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(71253327593),
+            '9999999999')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp(71253327593.67890),
+            '9999999999')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('71253327593'),
+            '9999999999')
+        self.assertEquals(
+            utils.normalize_delete_at_timestamp('71253327593.67890'),
+            '9999999999')
+        self.assertRaises(ValueError, utils.normalize_timestamp, '')
+        self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
+
     def test_backwards(self):
         # Test swift.common.utils.backward
 

test/unit/obj/test_server.py

@@ -2270,8 +2270,8 @@ class TestObjectController(unittest.TestCase):
             'DELETE', 2, 'a', 'c', 'o', req, 'sda1')
         self.assertEquals(
             given_args, [
-                'DELETE', '.expiring_objects', '0',
+                'DELETE', '.expiring_objects', '0000000000',
-                '2-a/c/o', None, None, None,
+                '0000000002-a/c/o', None, None, None,
                 HeaderKeyDict({
                     'x-timestamp': '1',
                     'x-trans-id': '123',
@@ -2296,7 +2296,8 @@ class TestObjectController(unittest.TestCase):
         self.object_controller.delete_at_update(
             'DELETE', -2, 'a', 'c', 'o', req, 'sda1')
         self.assertEquals(given_args, [
-            'DELETE', '.expiring_objects', '0', '0-a/c/o', None, None, None,
+            'DELETE', '.expiring_objects', '0000000000', '0000000000-a/c/o',
+            None, None, None,
             HeaderKeyDict({
                 'x-timestamp': '1',
                 'x-trans-id': '1234',
@@ -2352,7 +2353,8 @@ class TestObjectController(unittest.TestCase):
                                                 req, 'sda1')
         self.assertEquals(
             given_args, [
-                'PUT', '.expiring_objects', '0', '2-a/c/o', '127.0.0.1:1234',
+                'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o',
+                '127.0.0.1:1234',
                 '3', 'sdc1', HeaderKeyDict({
                     'x-size': '0',
                     'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
@@ -2404,7 +2406,8 @@ class TestObjectController(unittest.TestCase):
                                                 req, 'sda1')
         self.assertEquals(
             given_args, [
-                'DELETE', '.expiring_objects', '0', '2-a/c/o', None, None,
+                'DELETE', '.expiring_objects', '0000000000',
+                '0000000002-a/c/o', None, None,
                 None, HeaderKeyDict({
                     'x-timestamp': '1', 'x-trans-id': '1234',
                     'referer': 'DELETE http://localhost/v1/a/c/o'}),