From 150f338fc246a33b32b11cdd3abcc2eaacc73bd1 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Mon, 23 Dec 2013 17:57:56 +0100 Subject: [PATCH] Remove swiftclient dep on direct_client Partial Implements: blueprint remove-swiftclient-dependency Change-Id: I9af7150e5d21d50e5f880e57796314b8f05822d2 --- bin/swift-dispersion-report | 3 +- swift/account/reaper.py | 5 ++- swift/common/direct_client.py | 9 ++++- swift/common/exceptions.py | 54 +++++++++++++++++++++++++ swift/container/sync.py | 4 +- test/probe/test_container_failures.py | 3 +- test/probe/test_empty_device_handoff.py | 5 ++- test/probe/test_object_failures.py | 11 ++--- test/probe/test_object_handoff.py | 9 +++-- test/unit/account/test_reaper.py | 2 +- test/unit/common/test_direct_client.py | 6 +-- test/unit/common/test_exceptions.py | 13 ++++++ test/unit/container/test_sync.py | 2 +- 13 files changed, 102 insertions(+), 24 deletions(-) mode change 100644 => 100755 test/probe/test_empty_device_handoff.py diff --git a/bin/swift-dispersion-report b/bin/swift-dispersion-report index 3734b7c780..764075b408 100755 --- a/bin/swift-dispersion-report +++ b/bin/swift-dispersion-report @@ -28,8 +28,9 @@ from eventlet import GreenPool, hubs, patcher, Timeout from eventlet.pools import Pool from swift.common import direct_client -from swiftclient import ClientException, Connection, get_auth +from swiftclient import Connection, get_auth from swift.common.ring import Ring +from swift.common.exceptions import ClientException from swift.common.utils import compute_eta, get_time_units, config_true_value diff --git a/swift/account/reaper.py b/swift/account/reaper.py index ff39f07543..eb7084ffd6 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -25,8 +25,9 @@ from eventlet import GreenPool, sleep, Timeout import swift.common.db from swift.account.server import DATADIR from swift.account.backend import AccountBroker -from swift.common.direct_client import ClientException, \ - direct_delete_container, direct_delete_object, direct_get_container +from swift.common.direct_client import direct_delete_container, \ + direct_delete_object, direct_get_container +from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import get_logger, whataremyips, ismount, \ config_true_value diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 08dde9b524..b86ee4c2d6 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -26,13 +26,18 @@ from time import time from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect -from swiftclient import ClientException, json_loads +from swift.common.exceptions import ClientException from swift.common.utils import normalize_timestamp, FileLikeIter from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict from swift.common.utils import quote +try: + import simplejson as json +except ImportError: + import json + def _get_direct_account_container(path, stype, node, part, account, marker=None, limit=None, @@ -74,7 +79,7 @@ def _get_direct_account_container(path, stype, node, part, if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] - return resp_headers, json_loads(resp.read()) + return resp_headers, json.loads(resp.read()) def gen_headers(hdrs_in=None, add_ts=False): diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index 7b0f084441..baafd8dc08 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -133,3 +133,57 @@ class ReplicationException(Exception): class ReplicationLockTimeout(LockTimeout): pass + + +class ClientException(Exception): + + def __init__(self, msg, http_scheme='', http_host='', http_port='', + http_path='', http_query='', http_status=0, http_reason='', + http_device='', http_response_content=''): + Exception.__init__(self, msg) + self.msg = msg + self.http_scheme = http_scheme + self.http_host = http_host + self.http_port = http_port + self.http_path = http_path + self.http_query = http_query + self.http_status = http_status + self.http_reason = http_reason + self.http_device = http_device + self.http_response_content = http_response_content + + def __str__(self): + a = self.msg + b = '' + if self.http_scheme: + b += '%s://' % self.http_scheme + if self.http_host: + b += self.http_host + if self.http_port: + b += ':%s' % self.http_port + if self.http_path: + b += self.http_path + if self.http_query: + b += '?%s' % self.http_query + if self.http_status: + if b: + b = '%s %s' % (b, self.http_status) + else: + b = str(self.http_status) + if self.http_reason: + if b: + b = '%s %s' % (b, self.http_reason) + else: + b = '- %s' % self.http_reason + if self.http_device: + if b: + b = '%s: device %s' % (b, self.http_device) + else: + b = 'device %s' % self.http_device + if self.http_response_content: + if len(self.http_response_content) <= 60: + b += ' %s' % self.http_response_content + else: + b += ' [first 60 chars of response] %s' \ + % self.http_response_content[:60] + return b and '%s: %s' % (a, b) or a diff --git a/swift/container/sync.py b/swift/container/sync.py index db66e332c1..048efe3fb1 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -22,10 +22,10 @@ from eventlet import sleep, Timeout import swift.common.db from swift.container import server as container_server -from swiftclient import ClientException, delete_object, put_object, \ - quote +from swiftclient import delete_object, put_object, quote from swift.container.backend import ContainerBroker from swift.common.direct_client import direct_get_object +from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import audit_location_generator, get_logger, \ hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index c9df7ff660..fb1b68b693 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -25,6 +25,7 @@ from sqlite3 import connect from swiftclient import client from swift.common import direct_client +from swift.common.exceptions import ClientException from swift.common.utils import hash_path, readconf from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server @@ -107,7 +108,7 @@ class TestContainerFailures(TestCase): try: direct_client.direct_get_container(cnode, cpart, self.account, container1) - except client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) headers, containers = client.get_account(self.url, self.token) diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py old mode 100644 new mode 100755 index 4a08822d54..e42a0fddb8 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -25,6 +25,7 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.exceptions import ClientException from test.probe.common import kill_server, kill_servers, reset_environment,\ start_server from swift.common.utils import readconf @@ -129,7 +130,7 @@ class TestEmptyDevice(TestCase): try: direct_client.direct_get_object(onode, opart, self.account, container, obj) - except direct_client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) @@ -158,7 +159,7 @@ class TestEmptyDevice(TestCase): try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) - except direct_client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index c6dbccf5ae..339648ea4f 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -23,6 +23,7 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.exceptions import ClientException from swift.common.utils import hash_path, readconf from swift.obj.diskfile import write_metadata, read_metadata from test.probe.common import kill_servers, reset_environment @@ -93,7 +94,7 @@ class TestObjectFailures(TestCase): direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") - except client.ClientException as err: + except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_range_etag(self): @@ -116,7 +117,7 @@ class TestObjectFailures(TestCase): direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") - except client.ClientException as err: + except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_get(self): @@ -133,7 +134,7 @@ class TestObjectFailures(TestCase): container, obj, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") - except client.ClientException as err: + except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_head(self): @@ -150,7 +151,7 @@ class TestObjectFailures(TestCase): container, obj, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") - except client.ClientException as err: + except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_post(self): @@ -170,7 +171,7 @@ class TestObjectFailures(TestCase): conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") - except client.ClientException as err: + except ClientException as err: self.assertEquals(err.http_status, 404) def test_runner(self): diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 5619441d14..dd7b91cdae 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -21,6 +21,7 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client +from swift.common.exceptions import ClientException from test.probe.common import kill_server, kill_servers, reset_environment, \ start_server @@ -110,7 +111,7 @@ class TestObjectHandoff(TestCase): try: direct_client.direct_get_object(onode, opart, self.account, container, obj) - except direct_client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) # Run the extra server last so it'll remove its extra partition @@ -142,7 +143,7 @@ class TestObjectHandoff(TestCase): try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) - except direct_client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) @@ -151,7 +152,7 @@ class TestObjectHandoff(TestCase): exc = None try: client.head_object(self.url, self.token, container, obj) - except direct_client.ClientException as err: + except client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404) objs = [o['name'] for o in @@ -189,7 +190,7 @@ class TestObjectHandoff(TestCase): try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) - except direct_client.ClientException as err: + except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index 10438218af..5c9a3ebed3 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -25,8 +25,8 @@ from contextlib import nested from swift.account import reaper from swift.account.server import DATADIR +from swift.common.exceptions import ClientException from swift.common.utils import normalize_timestamp -from swift.common.direct_client import ClientException class FakeLogger(object): diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index 420c4c5843..4637de6b9d 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -20,6 +20,7 @@ import StringIO from hashlib import md5 from swift.common import direct_client +from swift.common.exceptions import ClientException from swiftclient import json_loads @@ -292,9 +293,8 @@ class TestDirectClient(unittest.TestCase): was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(500) - self.assertRaises(direct_client.ClientException, - direct_client.direct_put_object, node, part, account, - container, name, contents) + self.assertRaises(ClientException, direct_client.direct_put_object, + node, part, account, container, name, contents) direct_client.http_connect = was_http_connector diff --git a/test/unit/common/test_exceptions.py b/test/unit/common/test_exceptions.py index 04adfe2bdd..6ff77a55d5 100644 --- a/test/unit/common/test_exceptions.py +++ b/test/unit/common/test_exceptions.py @@ -32,6 +32,19 @@ class TestExceptions(unittest.TestCase): finally: exc.cancel() + def test_client_exception(self): + strerror = 'test: HTTP://random:888/randompath?foo=1 666 reason: ' \ + 'device /sdb1 content' + exc = exceptions.ClientException('test', http_scheme='HTTP', + http_host='random', + http_port=888, + http_path='/randompath', + http_query='foo=1', + http_status=666, + http_reason='reason', + http_device='/sdb1', + http_response_content='content') + self.assertEqual(str(exc), strerror) if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index 7a555fec58..c8b75078a8 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -22,7 +22,7 @@ import mock from test.unit import FakeLogger from swift.container import sync from swift.common import utils -from swiftclient import ClientException +from swift.common.exceptions import ClientException utils.HASH_PATH_SUFFIX = 'endcap'