Updated direct_client to match the changes in client

This commit is contained in:
gholt 2010-09-05 21:06:16 -07:00
parent 2edfd2b951
commit d0367fdf19
7 changed files with 55 additions and 55 deletions

@ -86,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
retries_done[0] += attempts - 1
found = True
if not estimated_objects:
estimated_objects = info[0]
estimated_objects = int(info['x-container-object-count'])
except ClientException, err:
if err.http_status not in (404, 507):
error_log('Giving up on /%s/%s/%s: %s' % (part, account,

@ -310,7 +310,7 @@ class AccountReaper(Daemon):
try:
objects = direct_get_container(node, part, account, container,
marker=marker, conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)
response_timeout=self.node_timeout)[1]
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
except ClientException, err:

@ -504,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None,
raise ClientException('Object GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
if resp_chunk_size:
def _object_body():
@ -550,10 +546,6 @@ def head_object(url, token, container, name, http_conn=None):
raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value

@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
:param container: container name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s' % (account, container)
with Timeout(conn_timeout):
@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count')), \
int(resp.getheader('x-container-bytes-used'))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def direct_get_container(node, part, account, container, marker=None,
@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None,
:param delimeter: delimeter for the query
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
"""
path = '/%s/%s' % (account, container)
qs = 'format=json'
@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def direct_delete_container(node, part, account, container, conn_timeout=5,
@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
'DELETE', path, headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException(
'Container server %s:%s direct DELETE %s gave status %s' %
@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return resp
def direct_head_object(node, part, account, container, obj, conn_timeout=5,
@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: tuple of (content-type, object size, last modified timestamp,
etag, metadata dictionary)
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length')), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def direct_get_object(node, part, account, container, obj, conn_timeout=5,
response_timeout=15):
response_timeout=15, resp_chunk_size=None):
"""
Get object directly from the object server.
@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: object
:param resp_chunk_size: if defined, chunk size of data to read.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
"""
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
with Timeout(response_timeout):
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException(
'Object server %s:%s direct GET %s gave status %s' %
(node['ip'], node['port'],
@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return (resp.getheader('content-type'),
int(resp.getheader('content-length')),
resp.getheader('last-modified'),
resp.getheader('etag').strip('"'),
metadata,
resp.read())
if resp_chunk_size:
def _object_body():
buf = resp.read(resp_chunk_size)
while buf:
yield buf
buf = resp.read(resp_chunk_size)
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def direct_delete_object(node, part, account, container, obj,
@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj,
'DELETE', path, headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException(
'Object server %s:%s direct DELETE %s gave status %s' %
@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return resp
def retry(func, *args, **kwargs):

@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
((cnode['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(not direct_client.direct_get_container(cnode, cpart,
self.account, container))
self.account, container)[1])
ps = []
for n in xrange(1, 5):
ps.append(Popen(['swift-object-updater',
@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
for p in ps:
p.wait()
objs = [o['name'] for o in direct_client.direct_get_container(cnode,
cpart, self.account, container)]
cpart, self.account, container)[1]]
self.assert_(obj in objs)

@ -81,7 +81,7 @@ class TestObjectHandoff(unittest.TestCase):
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart,
self.account, container)]
self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
@ -127,9 +127,9 @@ class TestObjectHandoff(unittest.TestCase):
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.post_object(self.url, self.token, container, obj,
headers={'x-object-meta-probe': 'value'})
ometadata = client.head_object(self.url, self.token, container, obj)
if ometadata.get('x-object-meta-probe') != 'value':
raise Exception('Metadata incorrect, was %s' % repr(ometadata))
oheaders = client.head_object(self.url, self.token, container, obj)
if oheaders.get('x-object-meta-probe') != 'value':
raise Exception('Metadata incorrect, was %s' % repr(oheaders))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
@ -144,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase):
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
ometadata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-2]
if ometadata.get('probe') == 'value':
oheaders = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[0]
if oheaders.get('x-object-meta-probe') == 'value':
raise Exception('Previously downed object server had the new '
'metadata when it should not have it')
# Run the extra server last so it'll remove it's extra partition
@ -160,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase):
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
ometadata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-2]
if ometadata.get('probe') != 'value':
oheaders = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[0]
if oheaders.get('x-object-meta-probe') != 'value':
raise Exception(
'Previously downed object server did not have the new metadata')
@ -182,7 +182,7 @@ class TestObjectHandoff(unittest.TestCase):
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)]
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %

@ -16,7 +16,6 @@
# TODO: Tests
import unittest
from swift.common import direct_client
class TestAuditor(unittest.TestCase):