openstack/swift
Code Issues Proposed changes

Consolidating and standardizing x-delete-at format

Browse Source 
Change-Id: Idc916da1c7fe1cc43a2c26f7f7ee1d4fcdd52c89
This commit is contained in:
gholt 2014-01-13 17:45:56 +00:00
parent d698c21ab3
commit a3f2400cba
5 changed files with 95 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
swift/common/utils.py
View File

@ -535,6 +535,29 @@ def normalize_timestamp(timestamp):
return "%016.05f" % (float(timestamp))
def normalize_delete_at_timestamp(timestamp):
"""
Format a timestamp (string or numeric) into a standardized
xxxxxxxxxx (10) format.
Note that timestamps less than 0000000000 are raised to
0000000000 and values greater than November 20th, 2286 at
17:46:39 UTC will be capped at that date and time, resulting in
no return value exceeding 9999999999.
This cap is because the expirer is already working through a
sorted list of strings that were all a length of 10. Adding
another digit would mess up the sort and cause the expirer to
break from processing early. By 2286, this problem will need to
be fixed, probably by creating an additional .expiring_objects
account to work from with 11 (or more) digit container names.
:param timestamp: unix timestamp
:returns: normalized timestamp as a string
"""
return '%010d' % min(max(0, float(timestamp)), 9999999999)
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path

15
swift/obj/server.py
View File

@ -29,7 +29,7 @@ from hashlib import md5
from eventlet import sleep, Timeout
from swift.common.utils import public, get_logger, \
config_true_value, timing_stats, replication
config_true_value, timing_stats, replication, normalize_delete_at_timestamp
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, \
check_float, check_utf8
@ -254,10 +254,7 @@ class ObjectController(object):
:param request: the original request driving the update
:param objdevice: device name that the object is in
"""
# Quick cap that will work from now until Sat Nov 20 17:46:39 2286
# At that time, Swift will be so popular and pervasive I will have
# created income for thousands of future programmers.
delete_at = max(min(delete_at, 9999999999), 0)
delete_at = normalize_delete_at_timestamp(delete_at)
updates = [(None, None)]
partition = None
@ -276,8 +273,8 @@ class ObjectController(object):
'best guess as to the container name for now.' % op)
# TODO(gholt): In a future release, change the above warning to
# a raised exception and remove the guess code below.
delete_at_container = str(
delete_at / self.expiring_objects_container_divisor *
delete_at_container = (
int(delete_at) / self.expiring_objects_container_divisor *
self.expiring_objects_container_divisor)
partition = headers_in.get('X-Delete-At-Partition', None)
hosts = headers_in.get('X-Delete-At-Host', '')
@ -300,8 +297,10 @@ class ObjectController(object):
# it will be ignored when the expirer eventually tries to issue the
# object DELETE later since the X-Delete-At value won't match up.
delete_at_container = str(
delete_at / self.expiring_objects_container_divisor *
int(delete_at) / self.expiring_objects_container_divisor *
self.expiring_objects_container_divisor)
delete_at_container = normalize_delete_at_timestamp(
delete_at_container)
for host, contdevice in updates:
self.async_update(

30
swift/proxy/controllers/obj.py
View File

@ -41,7 +41,7 @@ from eventlet.timeout import Timeout
from swift.common.utils import ContextPool, normalize_timestamp, \
config_true_value, public, json, csv_append, GreenthreadSafeIterator, \
quorum_size, GreenAsyncPile
quorum_size, GreenAsyncPile, normalize_delete_at_timestamp
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_metadata, check_object_creation, \
CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE
@ -549,7 +549,8 @@ class ObjectController(Controller):
return HTTPBadRequest(request=req,
content_type='text/plain',
body='Non-integer X-Delete-After')
req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
req.headers['x-delete-at'] = normalize_delete_at_timestamp(
time.time() + x_delete_after)
if self.app.object_post_as_copy:
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % (
@ -587,8 +588,9 @@ class ObjectController(Controller):
return HTTPNotFound(request=req)
if 'x-delete-at' in req.headers:
try:
x_delete_at = int(req.headers['x-delete-at'])
if x_delete_at < time.time():
x_delete_at = normalize_delete_at_timestamp(
int(req.headers['x-delete-at']))
if int(x_delete_at) < time.time():
return HTTPBadRequest(
body='X-Delete-At in past', request=req,
content_type='text/plain')
@ -597,9 +599,9 @@ class ObjectController(Controller):
content_type='text/plain',
body='Non-integer X-Delete-At')
req.environ.setdefault('swift.log_info', []).append(
'x-delete-at:%d' % x_delete_at)
delete_at_container = str(
x_delete_at /
'x-delete-at:%s' % x_delete_at)
delete_at_container = normalize_delete_at_timestamp(
int(x_delete_at) /
self.app.expiring_objects_container_divisor *
self.app.expiring_objects_container_divisor)
delete_at_part, delete_at_nodes = \
@ -777,7 +779,8 @@ class ObjectController(Controller):
return HTTPBadRequest(request=req,
content_type='text/plain',
body='Non-integer X-Delete-After')
req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
req.headers['x-delete-at'] = normalize_delete_at_timestamp(
time.time() + x_delete_after)
partition, nodes = self.app.object_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# do a HEAD request for container sync and checking object versions
@ -928,8 +931,9 @@ class ObjectController(Controller):
if 'x-delete-at' in req.headers:
try:
x_delete_at = int(req.headers['x-delete-at'])
if x_delete_at < time.time():
x_delete_at = normalize_delete_at_timestamp(
int(req.headers['x-delete-at']))
if int(x_delete_at) < time.time():
return HTTPBadRequest(
body='X-Delete-At in past', request=req,
content_type='text/plain')
@ -937,9 +941,9 @@ class ObjectController(Controller):
return HTTPBadRequest(request=req, content_type='text/plain',
body='Non-integer X-Delete-At')
req.environ.setdefault('swift.log_info', []).append(
'x-delete-at:%d' % x_delete_at)
delete_at_container = str(
x_delete_at /
'x-delete-at:%s' % x_delete_at)
delete_at_container = normalize_delete_at_timestamp(
int(x_delete_at) /
self.app.expiring_objects_container_divisor *
self.app.expiring_objects_container_divisor)
delete_at_part, delete_at_nodes = \

40
test/unit/common/test_utils.py
View File

@ -210,6 +210,46 @@ class TestUtils(unittest.TestCase):
self.assertRaises(ValueError, utils.normalize_timestamp, '')
self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
def test_normalize_delete_at_timestamp(self):
self.assertEquals(
utils.normalize_delete_at_timestamp(1253327593),
'1253327593')
self.assertEquals(
utils.normalize_delete_at_timestamp(1253327593.67890),
'1253327593')
self.assertEquals(
utils.normalize_delete_at_timestamp('1253327593'),
'1253327593')
self.assertEquals(
utils.normalize_delete_at_timestamp('1253327593.67890'),
'1253327593')
self.assertEquals(
utils.normalize_delete_at_timestamp(-1253327593),
'0000000000')
self.assertEquals(
utils.normalize_delete_at_timestamp(-1253327593.67890),
'0000000000')
self.assertEquals(
utils.normalize_delete_at_timestamp('-1253327593'),
'0000000000')
self.assertEquals(
utils.normalize_delete_at_timestamp('-1253327593.67890'),
'0000000000')
self.assertEquals(
utils.normalize_delete_at_timestamp(71253327593),
'9999999999')
self.assertEquals(
utils.normalize_delete_at_timestamp(71253327593.67890),
'9999999999')
self.assertEquals(
utils.normalize_delete_at_timestamp('71253327593'),
'9999999999')
self.assertEquals(
utils.normalize_delete_at_timestamp('71253327593.67890'),
'9999999999')
self.assertRaises(ValueError, utils.normalize_timestamp, '')
self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
def test_backwards(self):
# Test swift.common.utils.backward

13
test/unit/obj/test_server.py
View File

@ -2270,8 +2270,8 @@ class TestObjectController(unittest.TestCase):
'DELETE', 2, 'a', 'c', 'o', req, 'sda1')
self.assertEquals(
given_args, [
'DELETE', '.expiring_objects', '0',
'2-a/c/o', None, None, None,
'DELETE', '.expiring_objects', '0000000000',
'0000000002-a/c/o', None, None, None,
HeaderKeyDict({
'x-timestamp': '1',
'x-trans-id': '123',
@ -2296,7 +2296,8 @@ class TestObjectController(unittest.TestCase):
self.object_controller.delete_at_update(
'DELETE', -2, 'a', 'c', 'o', req, 'sda1')
self.assertEquals(given_args, [
'DELETE', '.expiring_objects', '0', '0-a/c/o', None, None, None,
'DELETE', '.expiring_objects', '0000000000', '0000000000-a/c/o',
None, None, None,
HeaderKeyDict({
'x-timestamp': '1',
'x-trans-id': '1234',
@ -2352,7 +2353,8 @@ class TestObjectController(unittest.TestCase):
req, 'sda1')
self.assertEquals(
given_args, [
'PUT', '.expiring_objects', '0', '2-a/c/o', '127.0.0.1:1234',
'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o',
'127.0.0.1:1234',
'3', 'sdc1', HeaderKeyDict({
'x-size': '0',
'x-etag': 'd41d8cd98f00b204e9800998ecf8427e',
@ -2404,7 +2406,8 @@ class TestObjectController(unittest.TestCase):
req, 'sda1')
self.assertEquals(
given_args, [
'DELETE', '.expiring_objects', '0', '2-a/c/o', None, None,
'DELETE', '.expiring_objects', '0000000000',
'0000000002-a/c/o', None, None,
None, HeaderKeyDict({
'x-timestamp': '1', 'x-trans-id': '1234',
'referer': 'DELETE http://localhost/v1/a/c/o'}),