Merge "py3: port direct_client"
This commit is contained in:
Zuul
Gerrit Code Review
commit
bd9d499b12
@ -54,6 +54,12 @@ class DirectClientException(ClientException):
|
||||
http_reason=resp.reason, http_headers=headers)
|
||||
|
||||
|
||||
def _make_path(*components):
|
||||
return u'/' + u'/'.join(
|
||||
x.decode('utf-8') if isinstance(x, six.binary_type) else x
|
||||
for x in components)
|
||||
|
||||
|
||||
def _make_req(node, part, method, path, headers, stype,
|
||||
conn_timeout=5, response_timeout=15, send_timeout=15,
|
||||
contents=None, content_length=None, chunk_size=65535):
|
||||
@ -105,9 +111,9 @@ def _make_req(node, part, method, path, headers, stype,
|
||||
if content_length is None:
|
||||
chunk = contents_f.read(chunk_size)
|
||||
while chunk:
|
||||
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
|
||||
conn.send(b'%x\r\n%s\r\n' % (len(chunk), chunk))
|
||||
chunk = contents_f.read(chunk_size)
|
||||
conn.send('0\r\n\r\n')
|
||||
conn.send(b'0\r\n\r\n')
|
||||
else:
|
||||
left = content_length
|
||||
while left > 0:
|
||||
@ -168,7 +174,7 @@ def _get_direct_account_container(path, stype, node, part,
|
||||
if resp.status == HTTP_NO_CONTENT:
|
||||
resp.read()
|
||||
return resp_headers, []
|
||||
return resp_headers, json.loads(resp.read())
|
||||
return resp_headers, json.loads(resp.read().decode('ascii'))
|
||||
|
||||
|
||||
def gen_headers(hdrs_in=None, add_ts=False, add_user_agent=True):
|
||||
@ -200,7 +206,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
|
||||
:returns: a tuple of (response headers, a list of containers) The response
|
||||
headers will HeaderKeyDict.
|
||||
"""
|
||||
path = '/' + account
|
||||
path = _make_path(account)
|
||||
return _get_direct_account_container(path, "Account", node, part,
|
||||
marker=marker,
|
||||
limit=limit, prefix=prefix,
|
||||
@ -216,7 +222,7 @@ def direct_delete_account(node, part, account, conn_timeout=5,
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
path = '/%s' % account
|
||||
path = _make_path(account)
|
||||
_make_req(node, part, 'DELETE', path, gen_headers(headers, True),
|
||||
'Account', conn_timeout, response_timeout)
|
||||
|
||||
@ -235,7 +241,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
|
||||
:returns: a dict containing the response's headers in a HeaderKeyDict
|
||||
:raises ClientException: HTTP HEAD request failed
|
||||
"""
|
||||
path = '/%s/%s' % (account, container)
|
||||
path = _make_path(account, container)
|
||||
resp = _make_req(node, part, 'HEAD', path, gen_headers(),
|
||||
'Container', conn_timeout, response_timeout)
|
||||
|
||||
@ -268,7 +274,7 @@ def direct_get_container(node, part, account, container, marker=None,
|
||||
:returns: a tuple of (response headers, a list of objects) The response
|
||||
headers will be a HeaderKeyDict.
|
||||
"""
|
||||
path = '/%s/%s' % (account, container)
|
||||
path = _make_path(account, container)
|
||||
return _get_direct_account_container(path, "Container", node,
|
||||
part, marker=marker,
|
||||
limit=limit, prefix=prefix,
|
||||
@ -297,7 +303,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
path = '/%s/%s' % (account, container)
|
||||
path = _make_path(account, container)
|
||||
add_timestamp = 'x-timestamp' not in (k.lower() for k in headers)
|
||||
_make_req(node, part, 'DELETE', path, gen_headers(headers, add_timestamp),
|
||||
'Container', conn_timeout, response_timeout)
|
||||
@ -328,7 +334,7 @@ def direct_put_container(node, part, account, container, conn_timeout=5,
|
||||
headers_out = gen_headers(headers,
|
||||
add_ts='x-timestamp' not in lower_headers,
|
||||
add_user_agent='user-agent' not in lower_headers)
|
||||
path = '/%s/%s' % (account, container)
|
||||
path = _make_path(account, container)
|
||||
_make_req(node, part, 'PUT', path, headers_out, 'Container', conn_timeout,
|
||||
response_timeout, contents=contents,
|
||||
content_length=content_length, chunk_size=chunk_size)
|
||||
@ -342,7 +348,7 @@ def direct_put_container_object(node, part, account, container, obj,
|
||||
|
||||
have_x_timestamp = 'x-timestamp' in (k.lower() for k in headers)
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, obj)
|
||||
path = _make_path(account, container, obj)
|
||||
_make_req(node, part, 'PUT', path,
|
||||
gen_headers(headers, add_ts=(not have_x_timestamp)),
|
||||
'Container', conn_timeout, response_timeout)
|
||||
@ -357,7 +363,7 @@ def direct_delete_container_object(node, part, account, container, obj,
|
||||
headers = gen_headers(headers, add_ts='x-timestamp' not in (
|
||||
k.lower() for k in headers))
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, obj)
|
||||
path = _make_path(account, container, obj)
|
||||
_make_req(node, part, 'DELETE', path, headers,
|
||||
'Container', conn_timeout, response_timeout)
|
||||
|
||||
@ -383,7 +389,7 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
|
||||
|
||||
headers = gen_headers(headers)
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, obj)
|
||||
path = _make_path(account, container, obj)
|
||||
resp = _make_req(node, part, 'HEAD', path, headers,
|
||||
'Object', conn_timeout, response_timeout)
|
||||
|
||||
@ -414,7 +420,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, obj)
|
||||
path = _make_path(account, container, obj)
|
||||
with Timeout(conn_timeout):
|
||||
conn = http_connect(node['ip'], node['port'], node['device'], part,
|
||||
'GET', path, headers=gen_headers(headers))
|
||||
@ -464,7 +470,7 @@ def direct_put_object(node, part, account, container, name, contents,
|
||||
:raises ClientException: HTTP PUT request failed
|
||||
"""
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, name)
|
||||
path = _make_path(account, container, name)
|
||||
if headers is None:
|
||||
headers = {}
|
||||
if etag:
|
||||
@ -499,7 +505,7 @@ def direct_post_object(node, part, account, container, name, headers,
|
||||
:param response_timeout: timeout in seconds for getting the response
|
||||
:raises ClientException: HTTP POST request failed
|
||||
"""
|
||||
path = '/%s/%s/%s' % (account, container, name)
|
||||
path = _make_path(account, container, name)
|
||||
_make_req(node, part, 'POST', path, gen_headers(headers, True),
|
||||
'Object', conn_timeout, response_timeout)
|
||||
|
||||
@ -524,7 +530,7 @@ def direct_delete_object(node, part, account, container, obj,
|
||||
headers = gen_headers(headers, add_ts='x-timestamp' not in (
|
||||
k.lower() for k in headers))
|
||||
|
||||
path = '/%s/%s/%s' % (account, container, obj)
|
||||
path = _make_path(account, container, obj)
|
||||
_make_req(node, part, 'DELETE', path, headers,
|
||||
'Object', conn_timeout, response_timeout)
|
||||
|
||||
|
||||
@ -665,8 +665,10 @@ class FileLikeIter(object):
|
||||
"""
|
||||
Wraps an iterable to behave as a file-like object.
|
||||
|
||||
The iterable must yield bytes strings.
|
||||
The iterable must be a byte string or yield byte strings.
|
||||
"""
|
||||
if isinstance(iterable, bytes):
|
||||
iterable = (iterable, )
|
||||
self.iterator = iter(iterable)
|
||||
self.buf = None
|
||||
self.closed = False
|
||||
|
||||
@ -72,7 +72,7 @@ class FakeConn(object):
|
||||
return self.resp_headers.items()
|
||||
|
||||
def read(self, amt=None):
|
||||
if isinstance(self.body, six.StringIO):
|
||||
if isinstance(self.body, six.BytesIO):
|
||||
return self.body.read(amt)
|
||||
elif amt is None:
|
||||
return self.body
|
||||
@ -95,11 +95,6 @@ def mocked_http_conn(*args, **kwargs):
|
||||
yield fake_conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def noop_timeout(duration):
|
||||
yield
|
||||
|
||||
|
||||
@patch_policies
|
||||
class TestDirectClient(unittest.TestCase):
|
||||
|
||||
@ -122,7 +117,14 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.account, self.container, self.obj))
|
||||
self.user_agent = 'direct-client %s' % os.getpid()
|
||||
|
||||
patcher = mock.patch.object(direct_client, 'Timeout', noop_timeout)
|
||||
class FakeTimeout(BaseException):
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, typ, value, tb):
|
||||
pass
|
||||
|
||||
patcher = mock.patch.object(direct_client, 'Timeout', FakeTimeout)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
@ -178,7 +180,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
'X-Timestamp': '1234567890',
|
||||
'X-PUT-Timestamp': '1234567890'})
|
||||
|
||||
body = '[{"count": 1, "bytes": 20971520, "name": "c1"}]'
|
||||
body = b'[{"count": 1, "bytes": 20971520, "name": "c1"}]'
|
||||
|
||||
with mocked_http_conn(200, stub_headers, body) as conn:
|
||||
resp_headers, resp = direct_client.direct_get_account(
|
||||
@ -189,7 +191,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.req_headers['user-agent'],
|
||||
self.user_agent)
|
||||
self.assertEqual(resp_headers, stub_headers)
|
||||
self.assertEqual(json.loads(body), resp)
|
||||
self.assertEqual(json.loads(body.decode('ascii')), resp)
|
||||
self.assertIn('format=json', conn.query_string)
|
||||
for k, v in req_params.items():
|
||||
if v is None:
|
||||
@ -216,28 +218,23 @@ class TestDirectClient(unittest.TestCase):
|
||||
stub_headers = {'X-Trans-Id': 'txb5f59485c578460f8be9e-0053478d09'}
|
||||
body = 'a server error has occurred'
|
||||
with mocked_http_conn(500, stub_headers, body):
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_get_account(self.node, self.part,
|
||||
self.account)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
expected_err_msg_parts = (
|
||||
'Account server %s:%s' % (self.node['ip'], self.node['port']),
|
||||
'GET %r' % self.account_path,
|
||||
'status 500',
|
||||
)
|
||||
for item in expected_err_msg_parts:
|
||||
self.assertTrue(
|
||||
item in str(err), '%r was not in "%s"' % (item, err))
|
||||
self.assertEqual(err.http_host, self.node['ip'])
|
||||
self.assertEqual(err.http_port, self.node['port'])
|
||||
self.assertEqual(err.http_device, self.node['device'])
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertEqual(err.http_reason, 'Internal Error')
|
||||
self.assertEqual(err.http_headers, stub_headers)
|
||||
self.assertIn(item, str(raised.exception))
|
||||
self.assertEqual(raised.exception.http_host, self.node['ip'])
|
||||
self.assertEqual(raised.exception.http_port, self.node['port'])
|
||||
self.assertEqual(raised.exception.http_device, self.node['device'])
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertEqual(raised.exception.http_reason, 'Internal Error')
|
||||
self.assertEqual(raised.exception.http_headers, stub_headers)
|
||||
|
||||
def test_direct_get_account_no_content_does_not_parse_body(self):
|
||||
headers = {
|
||||
@ -258,17 +255,13 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_get_account_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_get_account(
|
||||
self.node, self.part, self.account)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.method, 'GET')
|
||||
self.assertEqual(conn.path, self.account_path)
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('GET' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('GET' in str(raised.exception))
|
||||
|
||||
def test_direct_delete_account(self):
|
||||
part = '0'
|
||||
@ -296,17 +289,15 @@ class TestDirectClient(unittest.TestCase):
|
||||
account = 'a'
|
||||
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_delete_account(self.node, part, account)
|
||||
except ClientException as err:
|
||||
pass
|
||||
self.assertEqual(self.node['ip'], conn.host)
|
||||
self.assertEqual(self.node['port'], conn.port)
|
||||
self.assertEqual('DELETE', conn.method)
|
||||
self.assertEqual('/sda/0/a', conn.path)
|
||||
self.assertIn('X-Timestamp', conn.req_headers)
|
||||
self.assertIn('User-Agent', conn.req_headers)
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
|
||||
def test_direct_head_container(self):
|
||||
headers = HeaderKeyDict(key='value')
|
||||
@ -327,13 +318,9 @@ class TestDirectClient(unittest.TestCase):
|
||||
headers = HeaderKeyDict(key='value')
|
||||
|
||||
with mocked_http_conn(503, headers) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_head_container(
|
||||
self.node, self.part, self.account, self.container)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
# check request
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
@ -341,9 +328,9 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.path, self.container_path)
|
||||
|
||||
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
|
||||
self.assertEqual(err.http_status, 503)
|
||||
self.assertEqual(err.http_headers, headers)
|
||||
self.assertTrue('HEAD' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 503)
|
||||
self.assertEqual(raised.exception.http_headers, headers)
|
||||
self.assertTrue('HEAD' in str(raised.exception))
|
||||
|
||||
def test_direct_head_container_deleted(self):
|
||||
important_timestamp = Timestamp.now().internal
|
||||
@ -351,27 +338,23 @@ class TestDirectClient(unittest.TestCase):
|
||||
important_timestamp})
|
||||
|
||||
with mocked_http_conn(404, headers) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_head_container(
|
||||
self.node, self.part, self.account, self.container)
|
||||
except Exception as err:
|
||||
self.assertTrue(isinstance(err, ClientException))
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'HEAD')
|
||||
self.assertEqual(conn.path, self.container_path)
|
||||
|
||||
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
|
||||
self.assertEqual(err.http_status, 404)
|
||||
self.assertEqual(err.http_headers, headers)
|
||||
self.assertEqual(raised.exception.http_status, 404)
|
||||
self.assertEqual(raised.exception.http_headers, headers)
|
||||
|
||||
def test_direct_get_container(self):
|
||||
def do_test(req_params):
|
||||
headers = HeaderKeyDict({'key': 'value'})
|
||||
body = ('[{"hash": "8f4e3", "last_modified": "317260", '
|
||||
'"bytes": 209}]')
|
||||
body = (b'[{"hash": "8f4e3", "last_modified": "317260", '
|
||||
b'"bytes": 209}]')
|
||||
|
||||
with mocked_http_conn(200, headers, body) as conn:
|
||||
resp_headers, resp = direct_client.direct_get_container(
|
||||
@ -384,7 +367,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.req_headers['user-agent'],
|
||||
self.user_agent)
|
||||
self.assertEqual(headers, resp_headers)
|
||||
self.assertEqual(json.loads(body), resp)
|
||||
self.assertEqual(json.loads(body.decode('ascii')), resp)
|
||||
self.assertIn('format=json', conn.query_string)
|
||||
for k, v in req_params.items():
|
||||
if v is None:
|
||||
@ -443,24 +426,20 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_delete_container_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_delete_container(
|
||||
self.node, self.part, self.account, self.container)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'DELETE')
|
||||
self.assertEqual(conn.path, self.container_path)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('DELETE' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('DELETE' in str(raised.exception))
|
||||
|
||||
def test_direct_put_container(self):
|
||||
body = 'Let us begin with a quick introduction'
|
||||
body = b'Let us begin with a quick introduction'
|
||||
headers = {'x-foo': 'bar', 'Content-Length': str(len(body)),
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'my UA'}
|
||||
@ -484,7 +463,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertIsNone(rv)
|
||||
|
||||
def test_direct_put_container_chunked(self):
|
||||
body = 'Let us begin with a quick introduction'
|
||||
body = b'Let us begin with a quick introduction'
|
||||
headers = {'x-foo': 'bar', 'Content-Type': 'application/json'}
|
||||
|
||||
with mocked_http_conn(204) as conn:
|
||||
@ -501,7 +480,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertTrue('x-timestamp' in conn.req_headers)
|
||||
self.assertEqual('bar', conn.req_headers.get('x-foo'))
|
||||
self.assertNotIn('Content-Length', conn.req_headers)
|
||||
expected_sent = '%0x\r\n%s\r\n0\r\n\r\n' % (len(body), body)
|
||||
expected_sent = b'%0x\r\n%s\r\n0\r\n\r\n' % (len(body), body)
|
||||
self.assertEqual(md5(expected_sent).hexdigest(),
|
||||
conn.etag.hexdigest())
|
||||
self.assertIsNone(rv)
|
||||
@ -538,22 +517,18 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_put_container_object_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_put_container_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'PUT')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('PUT' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('PUT' in str(raised.exception))
|
||||
|
||||
def test_direct_delete_container_object(self):
|
||||
with mocked_http_conn(204) as conn:
|
||||
@ -568,22 +543,18 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_delete_container_obj_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_delete_container_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'DELETE')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('DELETE' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('DELETE' in str(raised.exception))
|
||||
|
||||
def test_direct_head_object(self):
|
||||
headers = HeaderKeyDict({'x-foo': 'bar'})
|
||||
@ -604,45 +575,38 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_head_object_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_head_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'HEAD')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('HEAD' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('HEAD' in str(raised.exception))
|
||||
|
||||
def test_direct_head_object_not_found(self):
|
||||
important_timestamp = Timestamp.now().internal
|
||||
stub_headers = {'X-Backend-Important-Timestamp': important_timestamp}
|
||||
with mocked_http_conn(404, headers=stub_headers) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_head_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'HEAD')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
|
||||
self.assertEqual(err.http_status, 404)
|
||||
self.assertEqual(err.http_headers['x-backend-important-timestamp'],
|
||||
important_timestamp)
|
||||
self.assertEqual(raised.exception.http_status, 404)
|
||||
self.assertEqual(
|
||||
raised.exception.http_headers['x-backend-important-timestamp'],
|
||||
important_timestamp)
|
||||
|
||||
def test_direct_get_object(self):
|
||||
contents = six.StringIO('123456')
|
||||
contents = six.BytesIO(b'123456')
|
||||
|
||||
with mocked_http_conn(200, body=contents) as conn:
|
||||
resp_header, obj_body = direct_client.direct_get_object(
|
||||
@ -655,41 +619,30 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_get_object_error(self):
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_get_object(
|
||||
self.node, self.part,
|
||||
self.account, self.container, self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'GET')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('GET' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('GET' in str(raised.exception))
|
||||
|
||||
def test_direct_get_object_chunks(self):
|
||||
contents = six.StringIO('123456')
|
||||
downloaded = b''
|
||||
contents = six.BytesIO(b'123456')
|
||||
|
||||
with mocked_http_conn(200, body=contents) as conn:
|
||||
resp_header, obj_body = direct_client.direct_get_object(
|
||||
self.node, self.part, self.account, self.container, self.obj,
|
||||
resp_chunk_size=2)
|
||||
while obj_body:
|
||||
try:
|
||||
chunk = obj_body.next()
|
||||
except StopIteration:
|
||||
break
|
||||
downloaded += chunk
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual('GET', conn.method)
|
||||
self.assertEqual(self.obj_path, conn.path)
|
||||
self.assertEqual('123456', downloaded)
|
||||
self.assertEqual([b'12', b'34', b'56'], list(obj_body))
|
||||
|
||||
def test_direct_post_object(self):
|
||||
headers = {'Key': 'value'}
|
||||
@ -712,14 +665,10 @@ class TestDirectClient(unittest.TestCase):
|
||||
headers = {'Key': 'value'}
|
||||
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_post_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj, headers)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'POST')
|
||||
@ -729,8 +678,8 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
|
||||
self.assertTrue('x-timestamp' in conn.req_headers)
|
||||
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertTrue('POST' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
self.assertTrue('POST' in str(raised.exception))
|
||||
|
||||
def test_direct_delete_object(self):
|
||||
with mocked_http_conn(200) as conn:
|
||||
@ -759,18 +708,14 @@ class TestDirectClient(unittest.TestCase):
|
||||
|
||||
def test_direct_delete_object_error(self):
|
||||
with mocked_http_conn(503) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_delete_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.method, 'DELETE')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
self.assertEqual(err.http_status, 503)
|
||||
self.assertTrue('DELETE' in str(err))
|
||||
self.assertEqual(raised.exception.http_status, 503)
|
||||
self.assertTrue('DELETE' in str(raised.exception))
|
||||
|
||||
def test_direct_get_suffix_hashes(self):
|
||||
data = {'a83': 'c130a2c17ed45102aada0f4eee69494ff'}
|
||||
@ -789,12 +734,12 @@ class TestDirectClient(unittest.TestCase):
|
||||
with self.assertRaises(DirectClientException) as cm:
|
||||
direct_client.direct_get_suffix_hashes(
|
||||
self.node, self.part, ['a83', 'b52'])
|
||||
self.assertIn('REPLICATE', cm.exception.message)
|
||||
self.assertIn('REPLICATE', cm.exception.args[0])
|
||||
self.assertIn(quote('/%s/%s/a83-b52'
|
||||
% (self.node['device'], self.part)),
|
||||
cm.exception.message)
|
||||
self.assertIn(self.node['replication_ip'], cm.exception.message)
|
||||
self.assertIn(self.node['replication_port'], cm.exception.message)
|
||||
cm.exception.args[0])
|
||||
self.assertIn(self.node['replication_ip'], cm.exception.args[0])
|
||||
self.assertIn(self.node['replication_port'], cm.exception.args[0])
|
||||
self.assertEqual(self.node['replication_ip'], cm.exception.http_host)
|
||||
self.assertEqual(self.node['replication_port'], cm.exception.http_port)
|
||||
self.assertEqual(self.node['device'], cm.exception.http_device)
|
||||
@ -807,7 +752,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self._test_direct_get_suffix_hashes_fail(507)
|
||||
|
||||
def test_direct_put_object_with_content_length(self):
|
||||
contents = six.StringIO('123456')
|
||||
contents = six.BytesIO(b'123456')
|
||||
|
||||
with mocked_http_conn(200) as conn:
|
||||
resp = direct_client.direct_put_object(
|
||||
@ -817,28 +762,24 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'PUT')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
self.assertEqual(md5('123456').hexdigest(), resp)
|
||||
self.assertEqual(md5(b'123456').hexdigest(), resp)
|
||||
|
||||
def test_direct_put_object_fail(self):
|
||||
contents = six.StringIO('123456')
|
||||
contents = six.BytesIO(b'123456')
|
||||
|
||||
with mocked_http_conn(500) as conn:
|
||||
try:
|
||||
with self.assertRaises(ClientException) as raised:
|
||||
direct_client.direct_put_object(
|
||||
self.node, self.part, self.account, self.container,
|
||||
self.obj, contents)
|
||||
except ClientException as err:
|
||||
pass
|
||||
else:
|
||||
self.fail('ClientException not raised')
|
||||
self.assertEqual(conn.host, self.node['ip'])
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'PUT')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
self.assertEqual(err.http_status, 500)
|
||||
self.assertEqual(raised.exception.http_status, 500)
|
||||
|
||||
def test_direct_put_object_chunked(self):
|
||||
contents = six.StringIO('123456')
|
||||
contents = six.BytesIO(b'123456')
|
||||
|
||||
with mocked_http_conn(200) as conn:
|
||||
resp = direct_client.direct_put_object(
|
||||
@ -848,7 +789,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual(conn.method, 'PUT')
|
||||
self.assertEqual(conn.path, self.obj_path)
|
||||
self.assertEqual(md5('6\r\n123456\r\n0\r\n\r\n').hexdigest(), resp)
|
||||
self.assertEqual(md5(b'6\r\n123456\r\n0\r\n\r\n').hexdigest(), resp)
|
||||
|
||||
def test_direct_put_object_args(self):
|
||||
# One test to cover all missing checks
|
||||
@ -863,10 +804,10 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(self.obj_path, conn.path)
|
||||
self.assertEqual(conn.req_headers['Content-Length'], '0')
|
||||
self.assertEqual(conn.req_headers['Content-Type'], 'Text')
|
||||
self.assertEqual(md5('0\r\n\r\n').hexdigest(), resp)
|
||||
self.assertEqual(md5(b'0\r\n\r\n').hexdigest(), resp)
|
||||
|
||||
def test_direct_put_object_header_content_length(self):
|
||||
contents = six.StringIO('123456')
|
||||
contents = six.BytesIO(b'123456')
|
||||
stub_headers = HeaderKeyDict({
|
||||
'Content-Length': '6'})
|
||||
|
||||
@ -878,7 +819,7 @@ class TestDirectClient(unittest.TestCase):
|
||||
self.assertEqual(conn.port, self.node['port'])
|
||||
self.assertEqual('PUT', conn.method)
|
||||
self.assertEqual(conn.req_headers['Content-length'], '6')
|
||||
self.assertEqual(md5('123456').hexdigest(), resp)
|
||||
self.assertEqual(md5(b'123456').hexdigest(), resp)
|
||||
|
||||
def test_retry(self):
|
||||
headers = HeaderKeyDict({'key': 'value'})
|
||||
@ -905,14 +846,11 @@ class TestDirectClient(unittest.TestCase):
|
||||
retries=2, error_log=logger.error)
|
||||
self.assertEqual('DELETE', conn.method)
|
||||
self.assertEqual(err_ctx.exception.http_status, 500)
|
||||
self.assertIn('DELETE', err_ctx.exception.message)
|
||||
self.assertIn(quote('/%s/%s/%s/%s/%s'
|
||||
% (self.node['device'].encode('utf-8'),
|
||||
self.part, self.account,
|
||||
self.container, self.obj)),
|
||||
err_ctx.exception.message)
|
||||
self.assertIn(self.node['ip'], err_ctx.exception.message)
|
||||
self.assertIn(self.node['port'], err_ctx.exception.message)
|
||||
self.assertIn('DELETE', err_ctx.exception.args[0])
|
||||
self.assertIn(self.obj_path,
|
||||
err_ctx.exception.args[0])
|
||||
self.assertIn(self.node['ip'], err_ctx.exception.args[0])
|
||||
self.assertIn(self.node['port'], err_ctx.exception.args[0])
|
||||
self.assertEqual(self.node['ip'], err_ctx.exception.http_host)
|
||||
self.assertEqual(self.node['port'], err_ctx.exception.http_port)
|
||||
self.assertEqual(self.node['device'], err_ctx.exception.http_device)
|
||||
|
||||
1
tox.ini
1
tox.ini
@ -49,6 +49,7 @@ commands =
|
||||
test/unit/common/test_bufferedhttp.py \
|
||||
test/unit/common/test_constraints.py \
|
||||
test/unit/common/test_daemon.py \
|
||||
test/unit/common/test_direct_client.py \
|
||||
test/unit/common/test_exceptions.py \
|
||||
test/unit/common/test_header_key_dict.py \
|
||||
test/unit/common/test_linkat.py \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user