Merge "Remove swiftclient dep on direct_client"

bin/swift-dispersion-report

@@ -28,8 +28,9 @@ from eventlet import GreenPool, hubs, patcher, Timeout
 from eventlet.pools import Pool
 
 from swift.common import direct_client
-from swiftclient import ClientException, Connection, get_auth
+from swiftclient import Connection, get_auth
 from swift.common.ring import Ring
+from swift.common.exceptions import ClientException
 from swift.common.utils import compute_eta, get_time_units, config_true_value
 
 

swift/account/reaper.py

@@ -25,8 +25,9 @@ from eventlet import GreenPool, sleep, Timeout
 import swift.common.db
 from swift.account.server import DATADIR
 from swift.account.backend import AccountBroker
-from swift.common.direct_client import ClientException, \
+from swift.common.direct_client import direct_delete_container, \
-    direct_delete_container, direct_delete_object, direct_get_container
+    direct_delete_object, direct_get_container
+from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, whataremyips, ismount, \
     config_true_value

swift/common/direct_client.py

@@ -26,13 +26,18 @@ from time import time
 from eventlet import sleep, Timeout
 
 from swift.common.bufferedhttp import http_connect
-from swiftclient import ClientException, json_loads
+from swift.common.exceptions import ClientException
 from swift.common.utils import normalize_timestamp, FileLikeIter
 from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
     is_success, is_server_error
 from swift.common.swob import HeaderKeyDict
 from swift.common.utils import quote
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
 
 def _get_direct_account_container(path, stype, node, part,
                                   account, marker=None, limit=None,
@@ -74,7 +79,7 @@ def _get_direct_account_container(path, stype, node, part,
     if resp.status == HTTP_NO_CONTENT:
         resp.read()
         return resp_headers, []
-    return resp_headers, json_loads(resp.read())
+    return resp_headers, json.loads(resp.read())
 
 
 def gen_headers(hdrs_in=None, add_ts=False):

swift/common/exceptions.py

@@ -133,3 +133,57 @@ class ReplicationException(Exception):
 
 class ReplicationLockTimeout(LockTimeout):
     pass
+
+
+class ClientException(Exception):
+
+    def __init__(self, msg, http_scheme='', http_host='', http_port='',
+                 http_path='', http_query='', http_status=0, http_reason='',
+                 http_device='', http_response_content=''):
+        Exception.__init__(self, msg)
+        self.msg = msg
+        self.http_scheme = http_scheme
+        self.http_host = http_host
+        self.http_port = http_port
+        self.http_path = http_path
+        self.http_query = http_query
+        self.http_status = http_status
+        self.http_reason = http_reason
+        self.http_device = http_device
+        self.http_response_content = http_response_content
+
+    def __str__(self):
+        a = self.msg
+        b = ''
+        if self.http_scheme:
+            b += '%s://' % self.http_scheme
+        if self.http_host:
+            b += self.http_host
+        if self.http_port:
+            b += ':%s' % self.http_port
+        if self.http_path:
+            b += self.http_path
+        if self.http_query:
+            b += '?%s' % self.http_query
+        if self.http_status:
+            if b:
+                b = '%s %s' % (b, self.http_status)
+            else:
+                b = str(self.http_status)
+        if self.http_reason:
+            if b:
+                b = '%s %s' % (b, self.http_reason)
+            else:
+                b = '- %s' % self.http_reason
+        if self.http_device:
+            if b:
+                b = '%s: device %s' % (b, self.http_device)
+            else:
+                b = 'device %s' % self.http_device
+        if self.http_response_content:
+            if len(self.http_response_content) <= 60:
+                b += '   %s' % self.http_response_content
+            else:
+                b += '  [first 60 chars of response] %s' \
+                    % self.http_response_content[:60]
+        return b and '%s: %s' % (a, b) or a

swift/container/sync.py

@@ -22,10 +22,10 @@ from eventlet import sleep, Timeout
 
 import swift.common.db
 from swift.container import server as container_server
-from swiftclient import ClientException, delete_object, put_object, \
+from swiftclient import delete_object, put_object, quote
-    quote
 from swift.container.backend import ContainerBroker
 from swift.common.direct_client import direct_get_object
+from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.utils import audit_location_generator, get_logger, \
     hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter

test/probe/test_container_failures.py

@@ -25,6 +25,7 @@ from sqlite3 import connect
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.exceptions import ClientException
 from swift.common.utils import hash_path, readconf
 from test.probe.common import get_to_final_state, kill_nonprimary_server, \
     kill_server, kill_servers, reset_environment, start_server
@@ -107,7 +108,7 @@ class TestContainerFailures(TestCase):
             try:
                 direct_client.direct_get_container(cnode, cpart, self.account,
                                                    container1)
-            except client.ClientException as err:
+            except ClientException as err:
                 exc = err
             self.assertEquals(exc.http_status, 404)
         headers, containers = client.get_account(self.url, self.token)

test/probe/test_empty_device_handoff.py

@@ -25,6 +25,7 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.exceptions import ClientException
 from test.probe.common import kill_server, kill_servers, reset_environment,\
     start_server
 from swift.common.utils import readconf
@@ -129,7 +130,7 @@ class TestEmptyDevice(TestCase):
         try:
             direct_client.direct_get_object(onode, opart, self.account,
                                             container, obj)
-        except direct_client.ClientException as err:
+        except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
         self.assertFalse(os.path.exists(obj_dir))
@@ -158,7 +159,7 @@ class TestEmptyDevice(TestCase):
         try:
             direct_client.direct_get_object(another_onode, opart, self.account,
                                             container, obj)
-        except direct_client.ClientException as err:
+        except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
 

test/probe/test_object_failures.py

@@ -23,6 +23,7 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.exceptions import ClientException
 from swift.common.utils import hash_path, readconf
 from swift.obj.diskfile import write_metadata, read_metadata
 from test.probe.common import kill_servers, reset_environment
@@ -93,7 +94,7 @@ class TestObjectFailures(TestCase):
             direct_client.direct_get_object(onode, opart, self.account,
                                             container, obj)
             raise Exception("Did not quarantine object")
-        except client.ClientException as err:
+        except ClientException as err:
             self.assertEquals(err.http_status, 404)
 
     def run_quarantine_range_etag(self):
@@ -116,7 +117,7 @@ class TestObjectFailures(TestCase):
             direct_client.direct_get_object(onode, opart, self.account,
                                             container, obj)
             raise Exception("Did not quarantine object")
-        except client.ClientException as err:
+        except ClientException as err:
             self.assertEquals(err.http_status, 404)
 
     def run_quarantine_zero_byte_get(self):
@@ -133,7 +134,7 @@ class TestObjectFailures(TestCase):
                                             container, obj, conn_timeout=1,
                                             response_timeout=1)
             raise Exception("Did not quarantine object")
-        except client.ClientException as err:
+        except ClientException as err:
             self.assertEquals(err.http_status, 404)
 
     def run_quarantine_zero_byte_head(self):
@@ -150,7 +151,7 @@ class TestObjectFailures(TestCase):
                                              container, obj, conn_timeout=1,
                                              response_timeout=1)
             raise Exception("Did not quarantine object")
-        except client.ClientException as err:
+        except ClientException as err:
             self.assertEquals(err.http_status, 404)
 
     def run_quarantine_zero_byte_post(self):
@@ -170,7 +171,7 @@ class TestObjectFailures(TestCase):
                 conn_timeout=1,
                 response_timeout=1)
             raise Exception("Did not quarantine object")
-        except client.ClientException as err:
+        except ClientException as err:
             self.assertEquals(err.http_status, 404)
 
     def test_runner(self):

test/probe/test_object_handoff.py

@@ -21,6 +21,7 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.exceptions import ClientException
 from test.probe.common import kill_server, kill_servers, reset_environment, \
     start_server
 
@@ -110,7 +111,7 @@ class TestObjectHandoff(TestCase):
         try:
             direct_client.direct_get_object(onode, opart, self.account,
                                             container, obj)
-        except direct_client.ClientException as err:
+        except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
         # Run the extra server last so it'll remove its extra partition
@@ -142,7 +143,7 @@ class TestObjectHandoff(TestCase):
         try:
             direct_client.direct_get_object(another_onode, opart, self.account,
                                             container, obj)
-        except direct_client.ClientException as err:
+        except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
 
@@ -151,7 +152,7 @@ class TestObjectHandoff(TestCase):
         exc = None
         try:
             client.head_object(self.url, self.token, container, obj)
-        except direct_client.ClientException as err:
+        except client.ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
         objs = [o['name'] for o in
@@ -189,7 +190,7 @@ class TestObjectHandoff(TestCase):
         try:
             direct_client.direct_get_object(another_onode, opart, self.account,
                                             container, obj)
-        except direct_client.ClientException as err:
+        except ClientException as err:
             exc = err
         self.assertEquals(exc.http_status, 404)
 

test/unit/account/test_reaper.py

@@ -25,8 +25,8 @@ from contextlib import nested
 
 from swift.account import reaper
 from swift.account.server import DATADIR
+from swift.common.exceptions import ClientException
 from swift.common.utils import normalize_timestamp
-from swift.common.direct_client import ClientException
 
 
 class FakeLogger(object):

test/unit/common/test_direct_client.py

@@ -20,6 +20,7 @@ import StringIO
 from hashlib import md5
 
 from swift.common import direct_client
+from swift.common.exceptions import ClientException
 from swiftclient import json_loads
 
 
@@ -292,9 +293,8 @@ class TestDirectClient(unittest.TestCase):
         was_http_connector = direct_client.http_connect
         direct_client.http_connect = mock_http_connect(500)
 
-        self.assertRaises(direct_client.ClientException,
+        self.assertRaises(ClientException, direct_client.direct_put_object,
-                          direct_client.direct_put_object, node, part, account,
+                          node, part, account, container, name, contents)
-                          container, name, contents)
 
         direct_client.http_connect = was_http_connector
 

test/unit/common/test_exceptions.py

@@ -32,6 +32,19 @@ class TestExceptions(unittest.TestCase):
         finally:
             exc.cancel()
 
+    def test_client_exception(self):
+        strerror = 'test: HTTP://random:888/randompath?foo=1 666 reason: ' \
+                   'device /sdb1   content'
+        exc = exceptions.ClientException('test', http_scheme='HTTP',
+                                         http_host='random',
+                                         http_port=888,
+                                         http_path='/randompath',
+                                         http_query='foo=1',
+                                         http_status=666,
+                                         http_reason='reason',
+                                         http_device='/sdb1',
+                                         http_response_content='content')
+        self.assertEqual(str(exc), strerror)
 
 if __name__ == '__main__':
     unittest.main()

test/unit/container/test_sync.py

@@ -22,7 +22,7 @@ import mock
 from test.unit import FakeLogger
 from swift.container import sync
 from swift.common import utils
-from swiftclient import ClientException
+from swift.common.exceptions import ClientException
 
 
 utils.HASH_PATH_SUFFIX = 'endcap'