openstack/swift
Code Issues Proposed changes

Allows removal of ACLs

Browse Source 
fixes bug 1049017

As per the bug report, the -remove- form of deletion only works with
 meta headers and not with x-container-read and x-container-write.

This patch by Yukihiro KAWADA detects the container acls and sends
them through to the backend empty like the other metadata values.

patch2 extends metadata-helper in ContainerController tests so that
the new functionality can be tested.

patch3 changes the k.lower().startswith() commands for read/write to
a single k.lower - thanks David :)

patch4 fixes PEP8

patch5 fixes more than two hundred pep8 errors. Except one,
 where the pep8 tool complains of overindent, or underindent
depending on the position on the line. No idea how to fix that one.

patch6 fixes the remaining pep8 error - thanks Darrell

Change-Id: I36c2dd72b4636136c5ce7db414bf2b61501090ad
This commit is contained in:
Tom Fifield 2012-10-07 14:28:41 +11:00
parent 1ef17d8284
commit 124e75bc1b
2 changed files with 492 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
swift/proxy/controllers/base.py
View File

@ -107,11 +107,17 @@ class Controller(object):
self.trans_id = '-' self.trans_id = '-'
def transfer_headers(self, src_headers, dst_headers): def transfer_headers(self, src_headers, dst_headers):
x_remove = 'x-remove-%s-meta-' % self.server_type.lower()
x_meta = 'x-%s-meta-' % self.server_type.lower() st = self.server_type.lower()
x_remove = 'x-remove-%s-meta-' % st
x_remove_read = 'x-remove-%s-read' % st
x_remove_write = 'x-remove-%s-write' % st
x_meta = 'x-%s-meta-' % st
dst_headers.update((k.lower().replace('-remove', '', 1), '') dst_headers.update((k.lower().replace('-remove', '', 1), '')
for k in src_headers for k in src_headers
if k.lower().startswith(x_remove)) if k.lower().startswith(x_remove) or
k.lower() in (x_remove_read, x_remove_write))
dst_headers.update((k.lower(), v) dst_headers.update((k.lower(), v)
for k, v in src_headers.iteritems() for k, v in src_headers.iteritems()
if k.lower() in self.pass_through_headers or if k.lower() in self.pass_through_headers or
@ -135,7 +141,8 @@ class Controller(object):
""" """
self.error_increment(node) self.error_increment(node)
self.app.logger.error(_('%(msg)s %(ip)s:%(port)s'), self.app.logger.error(_('%(msg)s %(ip)s:%(port)s'),
{'msg': msg, 'ip': node['ip'], 'port': node['port']}) {'msg': msg, 'ip': node['ip'],
'port': node['port']})
def exception_occurred(self, node, typ, additional_info): def exception_occurred(self, node, typ, additional_info):
""" """
@ -220,7 +227,8 @@ class Controller(object):
try: try:
with ConnectionTimeout(self.app.conn_timeout): with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'], conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD', path, headers) node['device'], partition, 'HEAD',
path, headers)
with Timeout(self.app.node_timeout): with Timeout(self.app.node_timeout):
resp = conn.getresponse() resp = conn.getresponse()
body = resp.read() body = resp.read()
@ -241,7 +249,8 @@ class Controller(object):
result_code = -1 result_code = -1
except (Exception, Timeout): except (Exception, Timeout):
self.exception_occurred(node, _('Account'), self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s') % path) _('Trying to get account info for %s')
% path)
if result_code == HTTP_NOT_FOUND and autocreate: if result_code == HTTP_NOT_FOUND and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH: if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None, None return None, None, None
@ -252,7 +261,7 @@ class Controller(object):
self.app.account_ring, partition, 'PUT', self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes)) path, [headers] * len(nodes))
if not is_success(resp.status_int): if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' % \ self.app.logger.warning('Could not autocreate account %r' %
path) path)
return None, None, None return None, None, None
result_code = HTTP_OK result_code = HTTP_OK
@ -262,7 +271,8 @@ class Controller(object):
else: else:
cache_timeout = self.app.recheck_account_existence * 0.1 cache_timeout = self.app.recheck_account_existence * 0.1
self.app.memcache.set(cache_key, self.app.memcache.set(cache_key,
{'status': result_code, 'container_count': container_count}, {'status': result_code,
'container_count': container_count},
timeout=cache_timeout) timeout=cache_timeout)
if result_code == HTTP_OK: if result_code == HTTP_OK:
return partition, nodes, container_count return partition, nodes, container_count

356
test/unit/proxy/test_server.py
View File

@ -151,8 +151,11 @@ def setup():
partition, nodes = prosrv.account_ring.get_nodes('a') partition, nodes = prosrv.account_ring.get_nodes('a')
for node in nodes: for node in nodes:
conn = swift.proxy.controllers.obj.http_connect(node['ip'], conn = swift.proxy.controllers.obj.http_connect(node['ip'],
node['port'], node['device'], partition, 'PUT', '/a', node['port'],
{'X-Timestamp': ts, 'x-trans-id': 'test'}) node['device'],
partition, 'PUT', '/a',
{'X-Timestamp': ts,
'x-trans-id': 'test'})
resp = conn.getresponse() resp = conn.getresponse()
assert(resp.status == 201) assert(resp.status == 201)
# Create container # Create container
@ -622,7 +625,9 @@ class TestProxyServer(unittest.TestCase):
def test_internal_method_request(self): def test_internal_method_request(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(), container_ring=FakeRing(), object_ring=FakeRing(), FakeMemcache(),
container_ring=FakeRing(),
object_ring=FakeRing(),
account_ring=FakeRing()) account_ring=FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
Request.blank('/v1/a', environ={'REQUEST_METHOD': '__init__'})) Request.blank('/v1/a', environ={'REQUEST_METHOD': '__init__'}))
@ -630,7 +635,9 @@ class TestProxyServer(unittest.TestCase):
def test_inexistent_method_request(self): def test_inexistent_method_request(self):
baseapp = proxy_server.Application({}, baseapp = proxy_server.Application({},
FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing(), FakeMemcache(),
container_ring=FakeRing(),
account_ring=FakeRing(),
object_ring=FakeRing()) object_ring=FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
Request.blank('/v1/a', environ={'REQUEST_METHOD': '!invalid'})) Request.blank('/v1/a', environ={'REQUEST_METHOD': '!invalid'}))
@ -644,7 +651,8 @@ class TestProxyServer(unittest.TestCase):
with save_globals(): with save_globals():
set_http_connect(200) set_http_connect(200)
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing()) object_ring=FakeRing())
req = Request.blank('/v1/a') req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize req.environ['swift.authorize'] = authorize
@ -659,7 +667,8 @@ class TestProxyServer(unittest.TestCase):
called[0] = True called[0] = True
return HTTPUnauthorized(request=req) return HTTPUnauthorized(request=req)
app = proxy_server.Application(None, FakeMemcache(), app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing()) object_ring=FakeRing())
req = Request.blank('/v1/a') req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize req.environ['swift.authorize'] = authorize
@ -671,7 +680,8 @@ class TestProxyServer(unittest.TestCase):
swift_dir = mkdtemp() swift_dir = mkdtemp()
try: try:
baseapp = proxy_server.Application({'swift_dir': swift_dir}, baseapp = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), FakeMemcache(), FakeLogger(),
FakeRing(), FakeRing(),
FakeRing()) FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
Request.blank('/', environ={'CONTENT_LENGTH': '-1'})) Request.blank('/', environ={'CONTENT_LENGTH': '-1'}))
@ -688,8 +698,10 @@ class TestProxyServer(unittest.TestCase):
swift_dir = mkdtemp() swift_dir = mkdtemp()
try: try:
baseapp = proxy_server.Application({'swift_dir': swift_dir, baseapp = proxy_server.Application({'swift_dir': swift_dir,
'deny_host_headers': 'invalid_host.com'}, 'deny_host_headers':
FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), 'invalid_host.com'},
FakeMemcache(), FakeLogger(),
FakeRing(), FakeRing(),
FakeRing()) FakeRing())
resp = baseapp.handle_request( resp = baseapp.handle_request(
Request.blank('/v1/a/c/o', Request.blank('/v1/a/c/o',
@ -698,11 +710,13 @@ class TestProxyServer(unittest.TestCase):
finally: finally:
rmtree(swift_dir, ignore_errors=True) rmtree(swift_dir, ignore_errors=True)
class TestObjectController(unittest.TestCase): class TestObjectController(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(), self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing()) object_ring=FakeRing())
monkey_patch_mimetools() monkey_patch_mimetools()
@ -714,7 +728,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0', req = Request.blank('/a/c/o',
headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
res = method(req) res = method(req)
@ -723,7 +738,8 @@ class TestObjectController(unittest.TestCase):
# repeat test # repeat test
set_http_connect(*statuses, **kwargs) set_http_connect(*statuses, **kwargs)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0', req = Request.blank('/a/c/o',
headers={'Content-Length': '0',
'Content-Type': 'text/plain'}) 'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
res = method(req) res = method(req)
@ -756,7 +772,8 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(headers[:len(exp)], exp) self.assertEqual(headers[:len(exp)], exp)
req = Request.blank(path, req = Request.blank(path,
environ={'REQUEST_METHOD': 'GET'}, environ={'REQUEST_METHOD': 'GET'},
headers={'Content-Type': 'application/octet-stream', headers={'Content-Type':
'application/octet-stream',
'X-Newest': 'true'}) 'X-Newest': 'true'})
res = req.get_response(prosrv) res = req.get_response(prosrv)
self.assertEqual(res.status_int, 200) self.assertEqual(res.status_int, 200)
@ -771,19 +788,23 @@ class TestObjectController(unittest.TestCase):
'container', 'object') 'container', 'object')
def test_content_type(filename, expected): def test_content_type(filename, expected):
# The three responses here are for account_info() (HEAD to account server), # The three responses here are for account_info() (HEAD to
# container_info() (HEAD to container server) and three calls to # account server), container_info() (HEAD to container server)
# _connect_put_node() (PUT to three object servers) # and three calls to _connect_put_node() (PUT to three object
# servers)
set_http_connect(201, 201, 201, 201, 201, set_http_connect(201, 201, 201, 201, 201,
give_content_type=lambda content_type: give_content_type=lambda content_type:
self.assertEquals(content_type, expected.next())) self.assertEquals(content_type,
expected.next()))
# We need into include a transfer-encoding to get past # We need into include a transfer-encoding to get past
# constraints.check_object_creation() # constraints.check_object_creation()
req = Request.blank('/a/c/%s' % filename, {}, headers={'transfer-encoding': 'chunked'}) req = Request.blank('/a/c/%s' % filename, {},
headers={'transfer-encoding': 'chunked'})
self.app.update_request(req) self.app.update_request(req)
self.app.memcache.store = {} self.app.memcache.store = {}
res = controller.PUT(req) res = controller.PUT(req)
# If we don't check the response here we could miss problems in PUT() # If we don't check the response here we could miss problems
# in PUT()
self.assertEquals(res.status_int, 201) self.assertEquals(res.status_int, 201)
test_content_type('test.jpg', iter(['', '', 'image/jpeg', test_content_type('test.jpg', iter(['', '', 'image/jpeg',
@ -799,7 +820,8 @@ class TestObjectController(unittest.TestCase):
with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp: with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
fp.write('foo/bar foo\n') fp.write('foo/bar foo\n')
ba = proxy_server.Application({'swift_dir': swift_dir}, ba = proxy_server.Application({'swift_dir': swift_dir},
FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), FakeMemcache(), FakeLogger(),
FakeRing(), FakeRing(),
FakeRing()) FakeRing())
self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0], self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0],
'foo/bar') 'foo/bar')
@ -856,7 +878,8 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {} self.app.memcache.store = {}
set_http_connect(*statuses) set_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', req = Request.blank('/a/c/o.jpg',
environ={'REQUEST_METHOD': 'PUT'}, body='some data') environ={'REQUEST_METHOD': 'PUT'},
body='some data')
self.app.update_request(req) self.app.update_request(req)
res = controller.PUT(req) res = controller.PUT(req)
expected = str(expected) expected = str(expected)
@ -906,8 +929,8 @@ class TestObjectController(unittest.TestCase):
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={ req = Request.blank('/a/c/o', {},
'Content-Type': 'foo/bar'}) headers={'Content-Type': 'foo/bar'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
expected = str(expected) expected = str(expected)
@ -928,8 +951,8 @@ class TestObjectController(unittest.TestCase):
def test_status_map(statuses, expected): def test_status_map(statuses, expected):
set_http_connect(*statuses) set_http_connect(*statuses)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={ req = Request.blank('/a/c/o', {},
'Content-Type': 'foo/bar'}) headers={'Content-Type': 'foo/bar'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
expected = str(expected) expected = str(expected)
@ -1006,12 +1029,18 @@ class TestObjectController(unittest.TestCase):
expected_timestamp) expected_timestamp)
# acct cont obj obj obj # acct cont obj obj obj
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3') '2', '3'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') '3', '2'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1') '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
'3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
None, None), None)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
None, '1'), '1')
def test_GET_newest(self): def test_GET_newest(self):
with save_globals(): with save_globals():
@ -1030,12 +1059,18 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(res.headers.get('last-modified'), self.assertEquals(res.headers.get('last-modified'),
expected_timestamp) expected_timestamp)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3') '2', '3'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') '3', '2'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1') '3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
'3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
None, None), None)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
None, '1'), '1')
with save_globals(): with save_globals():
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
@ -1053,11 +1088,16 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(res.headers.get('last-modified'), self.assertEquals(res.headers.get('last-modified'),
expected_timestamp) expected_timestamp)
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '1') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '1') '2', '3'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '1') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') '3', '2'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, '1', '2'), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
'3', '1'), '1')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
'3', '1'), '3')
test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
'1', '2'), None)
def test_POST_meta_val_len(self): def test_POST_meta_val_len(self):
with save_globals(): with save_globals():
@ -1203,8 +1243,10 @@ class TestObjectController(unittest.TestCase):
return '' return ''
req = Request.blank('/a/c/o', req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, environ={'REQUEST_METHOD': 'PUT',
headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) 'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
@ -1214,8 +1256,10 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
self.app.client_timeout = 0.1 self.app.client_timeout = 0.1
req = Request.blank('/a/c/o', req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, environ={'REQUEST_METHOD': 'PUT',
headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) 'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
set_http_connect(201, 201, 201) set_http_connect(201, 201, 201)
# obj obj obj # obj obj obj
@ -1246,8 +1290,10 @@ class TestObjectController(unittest.TestCase):
raise Exception('Disconnected') raise Exception('Disconnected')
req = Request.blank('/a/c/o', req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, environ={'REQUEST_METHOD': 'PUT',
headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) 'wsgi.input': SlowBody()},
headers={'Content-Length': '4',
'Content-Type': 'text/plain'})
self.app.update_request(req) self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
@ -1309,7 +1355,8 @@ class TestObjectController(unittest.TestCase):
dev['port'] = 1 dev['port'] = 1
req = Request.blank('/a/c/o', req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT'}, environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, headers={'Content-Length': '4',
'Content-Type': 'text/plain'},
body=' ') body=' ')
self.app.update_request(req) self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
@ -1321,7 +1368,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(201, 201, 201, slow=True) set_http_connect(201, 201, 201, slow=True)
req = Request.blank('/a/c/o', req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT'}, environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, headers={'Content-Length': '4',
'Content-Type': 'text/plain'},
body=' ') body=' ')
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
@ -1332,9 +1380,11 @@ class TestObjectController(unittest.TestCase):
try: try:
self.app.object_ring.max_more_nodes = 2 self.app.object_ring.max_more_nodes = 2
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container',
'object')
partition, nodes = self.app.object_ring.get_nodes('account', partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object') 'container',
'object')
collected_nodes = [] collected_nodes = []
for node in controller.iter_nodes(partition, nodes, for node in controller.iter_nodes(partition, nodes,
self.app.object_ring): self.app.object_ring):
@ -1343,9 +1393,11 @@ class TestObjectController(unittest.TestCase):
self.app.object_ring.max_more_nodes = 20 self.app.object_ring.max_more_nodes = 20
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container',
'object')
partition, nodes = self.app.object_ring.get_nodes('account', partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object') 'container',
'object')
collected_nodes = [] collected_nodes = []
for node in controller.iter_nodes(partition, nodes, for node in controller.iter_nodes(partition, nodes,
self.app.object_ring): self.app.object_ring):
@ -1356,9 +1408,11 @@ class TestObjectController(unittest.TestCase):
self.app.logger = FakeLogger() self.app.logger = FakeLogger()
self.app.object_ring.max_more_nodes = 2 self.app.object_ring.max_more_nodes = 2
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container',
'object')
partition, nodes = self.app.object_ring.get_nodes('account', partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object') 'container',
'object')
collected_nodes = [] collected_nodes = []
for node in controller.iter_nodes(partition, nodes, for node in controller.iter_nodes(partition, nodes,
self.app.object_ring): self.app.object_ring):
@ -1373,9 +1427,11 @@ class TestObjectController(unittest.TestCase):
self.app.logger = FakeLogger() self.app.logger = FakeLogger()
self.app.object_ring.max_more_nodes = 2 self.app.object_ring.max_more_nodes = 2
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container',
'object')
partition, nodes = self.app.object_ring.get_nodes('account', partition, nodes = self.app.object_ring.get_nodes('account',
'container', 'object') 'container',
'object')
collected_nodes = [] collected_nodes = []
for node in controller.iter_nodes(partition, nodes, for node in controller.iter_nodes(partition, nodes,
self.app.object_ring): self.app.object_ring):
@ -1393,7 +1449,9 @@ class TestObjectController(unittest.TestCase):
'Object') 'Object')
self.assertEquals(resp.etag, None) self.assertEquals(resp.etag, None)
resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
'Object', etag='68b329da9893e34099c7d8ad5cb9c940') 'Object',
etag='68b329da9893e34099c7d8ad5cb9c940'
)
self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940') self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940')
def test_proxy_passes_content_type(self): def test_proxy_passes_content_type(self):
@ -1435,26 +1493,33 @@ class TestObjectController(unittest.TestCase):
set_shuffle() set_shuffle()
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container', 'object')
self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), 200) self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200),
200)
print controller.app.object_ring.devs print controller.app.object_ring.devs
self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2) self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2)
self.assert_('last_error' in controller.app.object_ring.devs[0]) self.assert_('last_error' in controller.app.object_ring.devs[0])
for _junk in xrange(self.app.error_suppression_limit): for _junk in xrange(self.app.error_suppression_limit):
self.assert_status_map(controller.HEAD, (200, 200, 503, 503, 503), 503) self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
503), 503)
self.assertEquals(controller.app.object_ring.devs[0]['errors'], self.assertEquals(controller.app.object_ring.devs[0]['errors'],
self.app.error_suppression_limit + 1) self.app.error_suppression_limit + 1)
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 503) self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
503)
self.assert_('last_error' in controller.app.object_ring.devs[0]) self.assert_('last_error' in controller.app.object_ring.devs[0])
self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201, 201), 503) self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201,
201), 503)
self.assert_status_map(controller.POST, self.assert_status_map(controller.POST,
(200, 200, 200, 200, 200, 200, 202, 202, 202), 503) (200, 200, 200, 200, 200, 200, 202, 202,
202), 503)
self.assert_status_map(controller.DELETE, self.assert_status_map(controller.DELETE,
(200, 200, 200, 204, 204, 204), 503) (200, 200, 200, 204, 204, 204), 503)
self.app.error_suppression_interval = -300 self.app.error_suppression_interval = -300
self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 200) self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
200)
self.assertRaises(BaseException, self.assertRaises(BaseException,
self.assert_status_map, controller.DELETE, self.assert_status_map, controller.DELETE,
(200, 200, 200, 204, 204, 204), 503, raise_exc=True) (200, 200, 200, 204, 204, 204), 503,
raise_exc=True)
def test_acc_or_con_missing_returns_404(self): def test_acc_or_con_missing_returns_404(self):
with save_globals(): with save_globals():
@ -1918,8 +1983,9 @@ class TestObjectController(unittest.TestCase):
req.account = 'a' req.account = 'a'
controller.object_name = 'o' controller.object_name = 'o'
set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, set_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
# acct cont objc objc objc obj obj obj #act cont objc objc objc obj obj obj
timestamps=('1', '1', '1', '3', '2', '4', '4', '4')) timestamps=('1', '1', '1', '3', '2', '4', '4',
'4'))
self.app.memcache.store = {} self.app.memcache.store = {}
resp = controller.COPY(req) resp = controller.COPY(req)
self.assertEquals(resp.status_int, 201) self.assertEquals(resp.status_int, 201)
@ -1978,8 +2044,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_bad_version(self): def test_chunked_put_bad_version(self):
# Check bad version # Check bad version
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n' fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n'
@ -1991,8 +2057,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_bad_path(self): def test_chunked_put_bad_path(self):
# Check bad path # Check bad path
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n' fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n'
@ -2004,8 +2070,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_bad_utf8(self): def test_chunked_put_bad_utf8(self):
# Check invalid utf-8 # Check invalid utf-8
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n' fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n'
@ -2018,8 +2084,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_bad_path_no_controller(self): def test_chunked_put_bad_path_no_controller(self):
# Check bad path, no controller # Check bad path, no controller
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n' fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n'
@ -2032,8 +2098,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_bad_method(self): def test_chunked_put_bad_method(self):
# Check bad method # Check bad method
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n' fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n'
@ -2046,10 +2112,10 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_unhandled_exception(self): def test_chunked_put_unhandled_exception(self):
# Check unhandled exception # Check unhandled exception
(prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv,
_test_servers obj2srv) = _test_servers
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
orig_update_request = prosrv.update_request orig_update_request = prosrv.update_request
def broken_update_request(*args, **kwargs): def broken_update_request(*args, **kwargs):
@ -2071,8 +2137,8 @@ class TestObjectController(unittest.TestCase):
# Head account, just a double check and really is here to test # Head account, just a double check and really is here to test
# the part Application.log_request that 'enforces' a # the part Application.log_request that 'enforces' a
# content_length on the response. # content_length on the response.
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n'
@ -2094,8 +2160,8 @@ class TestObjectController(unittest.TestCase):
'\xbf\x86.Test' '\xbf\x86.Test'
ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest' ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest'
# Create ustr container # Create ustr container
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n'
@ -2205,8 +2271,8 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put_chunked_put(self): def test_chunked_put_chunked_put(self):
# Do chunked object put # Do chunked object put
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
# Also happens to assert that x-storage-token is taken as a # Also happens to assert that x-storage-token is taken as a
@ -2235,8 +2301,8 @@ class TestObjectController(unittest.TestCase):
def test_version_manifest(self): def test_version_manifest(self):
versions_to_create = 3 versions_to_create = 3
# Create a container for our versioned object testing # Create a container for our versioned object testing
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
_test_sockets obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('PUT /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n' fd.write('PUT /v1/a/versions HTTP/1.1\r\nHost: localhost\r\n'
@ -2294,7 +2360,8 @@ class TestObjectController(unittest.TestCase):
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: ' fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
'\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200' exp = 'HTTP/1.1 200'
@ -2363,8 +2430,8 @@ class TestObjectController(unittest.TestCase):
for segment in xrange(versions_to_create - 1, 0, -1): for segment in xrange(versions_to_create - 1, 0, -1):
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: ' fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r'
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n\r\n') '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 2' # 2xx series response exp = 'HTTP/1.1 2' # 2xx series response
@ -2372,8 +2439,8 @@ class TestObjectController(unittest.TestCase):
# Ensure retrieving the manifest file gets the latest version # Ensure retrieving the manifest file gets the latest version
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: ' fd.write('GET /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') 'Connection: close\r\nX-Auth-Token: t\r\n\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200' exp = 'HTTP/1.1 200'
@ -2386,7 +2453,8 @@ class TestObjectController(unittest.TestCase):
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: ' fd.write('GET /v1/a/vers?prefix=004name/ HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n') 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r'
'\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 2' # 2xx series response exp = 'HTTP/1.1 2' # 2xx series response
@ -2407,8 +2475,8 @@ class TestObjectController(unittest.TestCase):
# delete the last verision # delete the last verision
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: ' fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n\r\n') 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 2' # 2xx series response exp = 'HTTP/1.1 2' # 2xx series response
@ -2487,8 +2555,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(headers[:len(exp)], exp) self.assertEquals(headers[:len(exp)], exp)
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: ' fd.write('DELETE /v1/a/versions/name HTTP/1.1\r\nHost: localhost\r\n'
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n\r\n') 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 2' # 2xx series response exp = 'HTTP/1.1 2' # 2xx series response
@ -2538,8 +2606,8 @@ class TestObjectController(unittest.TestCase):
# Delete the object # Delete the object
sock = connect_tcp(('localhost', prolis.getsockname()[1])) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile() fd = sock.makefile()
fd.write('DELETE /v1/a/whoops/foo HTTP/1.1\r\nHost: ' fd.write('DELETE /v1/a/whoops/foo HTTP/1.1\r\nHost: localhost\r\n'
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n\r\n') 'Connection: close\r\nX-Storage-Token: t\r\n\r\n')
fd.flush() fd.flush()
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 2' # 2xx response exp = 'HTTP/1.1 2' # 2xx response
@ -3104,7 +3172,8 @@ class TestObjectController(unittest.TestCase):
t = time() t = time()
proxy_server.time.time = lambda: t proxy_server.time.time = lambda: t
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60'}) headers={'Content-Type': 'foo/bar',
'X-Delete-After': '60'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status, '202 Fake') self.assertEquals(res.status, '202 Fake')
@ -3113,11 +3182,13 @@ class TestObjectController(unittest.TestCase):
self.app.object_post_as_copy = False self.app.object_post_as_copy = False
controller = proxy_server.ObjectController(self.app, 'account', controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object') 'container',
'object')
set_http_connect(200, 200, 202, 202, 202) set_http_connect(200, 200, 202, 202, 202)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60'}) headers={'Content-Type': 'foo/bar',
'X-Delete-After': '60'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status, '202 Fake') self.assertEquals(res.status, '202 Fake')
@ -3133,7 +3204,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60.1'}) headers={'Content-Type': 'foo/bar',
'X-Delete-After': '60.1'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status, '400 Bad Request') self.assertEquals(res.status, '400 Bad Request')
@ -3146,7 +3218,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) set_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-After': '-60'}) headers={'Content-Type': 'foo/bar',
'X-Delete-After': '-60'})
self.app.update_request(req) self.app.update_request(req)
res = controller.POST(req) res = controller.POST(req)
self.assertEquals(res.status, '400 Bad Request') self.assertEquals(res.status, '400 Bad Request')
@ -3168,7 +3241,8 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {} self.app.memcache.store = {}
t = str(int(time() + 100)) t = str(int(time() + 100))
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
controller.POST(req) controller.POST(req)
self.assertEquals(given_headers.get('X-Delete-At'), t) self.assertEquals(given_headers.get('X-Delete-At'), t)
@ -3178,7 +3252,8 @@ class TestObjectController(unittest.TestCase):
t = str(int(time() + 100)) + '.1' t = str(int(time() + 100)) + '.1'
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
resp = controller.POST(req) resp = controller.POST(req)
self.assertEquals(resp.status_int, 400) self.assertEquals(resp.status_int, 400)
@ -3186,7 +3261,8 @@ class TestObjectController(unittest.TestCase):
t = str(int(time() - 100)) t = str(int(time() - 100))
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
resp = controller.POST(req) resp = controller.POST(req)
self.assertEquals(resp.status_int, 400) self.assertEquals(resp.status_int, 400)
@ -3203,7 +3279,8 @@ class TestObjectController(unittest.TestCase):
t = time() t = time()
proxy_server.time.time = lambda: t proxy_server.time.time = lambda: t
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-After': '60'}) 'X-Delete-After': '60'})
self.app.update_request(req) self.app.update_request(req)
res = controller.PUT(req) res = controller.PUT(req)
@ -3220,7 +3297,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200, 201, 201, 201) set_http_connect(200, 200, 201, 201, 201)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-After': '60.1'}) 'X-Delete-After': '60.1'})
self.app.update_request(req) self.app.update_request(req)
res = controller.PUT(req) res = controller.PUT(req)
@ -3234,7 +3312,8 @@ class TestObjectController(unittest.TestCase):
set_http_connect(200, 200, 201, 201, 201) set_http_connect(200, 200, 201, 201, 201)
self.app.memcache.store = {} self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-After': '-60'}) 'X-Delete-After': '-60'})
self.app.update_request(req) self.app.update_request(req)
res = controller.PUT(req) res = controller.PUT(req)
@ -3256,7 +3335,8 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {} self.app.memcache.store = {}
t = str(int(time() + 100)) t = str(int(time() + 100))
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-At': t}) 'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
controller.PUT(req) controller.PUT(req)
@ -3267,7 +3347,8 @@ class TestObjectController(unittest.TestCase):
t = str(int(time() + 100)) + '.1' t = str(int(time() + 100)) + '.1'
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-At': t}) 'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
@ -3276,7 +3357,8 @@ class TestObjectController(unittest.TestCase):
t = str(int(time() - 100)) t = str(int(time() - 100))
req = Request.blank('/a/c/o', {}, req = Request.blank('/a/c/o', {},
headers={'Content-Length': '0', 'Content-Type': 'foo/bar', headers={'Content-Length': '0',
'Content-Type': 'foo/bar',
'X-Delete-At': t}) 'X-Delete-At': t})
self.app.update_request(req) self.app.update_request(req)
resp = controller.PUT(req) resp = controller.PUT(req)
@ -3321,12 +3403,14 @@ class TestObjectController(unittest.TestCase):
sock.close() sock.close()
self.assertEquals(before_request_instances, _request_instances) self.assertEquals(before_request_instances, _request_instances)
class TestContainerController(unittest.TestCase): class TestContainerController(unittest.TestCase):
"Test swift.proxy_server.ContainerController" "Test swift.proxy_server.ContainerController"
def setUp(self): def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(), self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing()) object_ring=FakeRing())
def assert_status_map(self, method, statuses, expected, def assert_status_map(self, method, statuses, expected,
@ -3448,7 +3532,8 @@ class TestContainerController(unittest.TestCase):
del dev['errors'] del dev['errors']
del dev['last_error'] del dev['last_error']
controller = proxy_server.ContainerController(self.app, controller = proxy_server.ContainerController(self.app,
'account', 'container') 'account',
'container')
if meth == 'PUT': if meth == 'PUT':
set_http_connect(200, 200, 200, 200, 200, 200, set_http_connect(200, 200, 200, 200, 200, 200,
missing_container=True) missing_container=True)
@ -3585,7 +3670,11 @@ class TestContainerController(unittest.TestCase):
for test_header, test_value in ( for test_header, test_value in (
('X-Container-Meta-TestHeader', 'TestValue'), ('X-Container-Meta-TestHeader', 'TestValue'),
('X-Container-Meta-TestHeader', ''), ('X-Container-Meta-TestHeader', ''),
('X-Remove-Container-Meta-TestHeader', 'anything')): ('X-Remove-Container-Meta-TestHeader', 'anything'),
('X-Container-Read', '.r:*'),
('X-Remove-Container-Read', 'anything'),
('X-Container-Write', 'anyone'),
('X-Remove-Container-Write', 'anything')):
test_errors = [] test_errors = []
def test_connect(ipaddr, port, device, partition, method, path, def test_connect(ipaddr, port, device, partition, method, path,
@ -3808,7 +3897,8 @@ class TestAccountController(unittest.TestCase):
def setUp(self): def setUp(self):
self.app = proxy_server.Application(None, FakeMemcache(), self.app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(), account_ring=FakeRing(),
container_ring=FakeRing(),
object_ring=FakeRing) object_ring=FakeRing)
def assert_status_map(self, method, statuses, expected): def assert_status_map(self, method, statuses, expected):
@ -3859,7 +3949,6 @@ class TestAccountController(unittest.TestCase):
self.assert_status_map(controller.GET, self.assert_status_map(controller.GET,
(404, 404, 404, 409, 409, 409, 409), 409) (404, 404, 404, 409, 409, 409, 409), 409)
def test_HEAD(self): def test_HEAD(self):
with save_globals(): with save_globals():
controller = proxy_server.AccountController(self.app, 'account') controller = proxy_server.AccountController(self.app, 'account')
@ -4221,6 +4310,7 @@ class TestSegmentedIterable(unittest.TestCase):
def test_load_next_segment_rate_limiting(self): def test_load_next_segment_rate_limiting(self):
sleep_calls = [] sleep_calls = []
def _stub_sleep(sleepy_time): def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time) sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep orig_sleep = swift.proxy.controllers.obj.sleep
@ -4231,22 +4321,25 @@ class TestSegmentedIterable(unittest.TestCase):
{'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'}, {'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
{'name': 'o4'}, {'name': 'o5'}]) {'name': 'o4'}, {'name': 'o5'}])
# rate_limit_after_segment == 3, so the first 3 segments should invoke # rate_limit_after_segment == 3, so the first 3 segments should
# no sleeping. # invoke no sleeping.
for _ in xrange(3): for _ in xrange(3):
segit._load_next_segment() segit._load_next_segment()
self.assertEquals([], sleep_calls) self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o3') self.assertEquals(self.controller.GETorHEAD_base_args[4],
'/a/lc/o3')
# Loading of next (4th) segment starts rate-limiting. # Loading of next (4th) segment starts rate-limiting.
segit._load_next_segment() segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2) self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o4') self.assertEquals(self.controller.GETorHEAD_base_args[4],
'/a/lc/o4')
sleep_calls = [] sleep_calls = []
segit._load_next_segment() segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2) self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[4], '/a/lc/o5') self.assertEquals(self.controller.GETorHEAD_base_args[4],
'/a/lc/o5')
finally: finally:
swift.proxy.controllers.obj.sleep = orig_sleep swift.proxy.controllers.obj.sleep = orig_sleep
@ -4280,8 +4373,8 @@ class TestSegmentedIterable(unittest.TestCase):
self.controller.GETorHEAD_base = local_GETorHEAD_base self.controller.GETorHEAD_base = local_GETorHEAD_base
self.assertRaises(Exception, self.assertRaises(Exception,
SegmentedIterable(self.controller, 'lc', [{'name': SegmentedIterable(self.controller, 'lc',
'o1'}])._load_next_segment) [{'name': 'o1'}])._load_next_segment)
self.assert_(self.controller.exception_args[0].startswith( self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest')) 'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]), self.assertEquals(str(self.controller.exception_info[1]),
@ -4328,7 +4421,8 @@ class TestSegmentedIterable(unittest.TestCase):
# Iterator value isn't a dict # Iterator value isn't a dict
self.assertRaises(Exception, self.assertRaises(Exception,
SegmentedIterable(self.controller, None, SegmentedIterable(self.controller, None,
[None]).app_iter_range(None, None).next) [None]).app_iter_range(None,
None).next)
self.assert_(self.controller.exception_args[0].startswith( self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest')) 'ERROR: While processing manifest'))
@ -4380,8 +4474,8 @@ class TestSegmentedIterable(unittest.TestCase):
def test_app_iter_range_with_many_segments(self): def test_app_iter_range_with_many_segments(self):
listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}, listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2},
{'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4}, {'name': {'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4},
'o5', 'bytes': 5}] {'name': 'o5', 'bytes': 5}]
segit = SegmentedIterable(self.controller, 'lc', listing) segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub() segit.response = Stub()