Config option to lower the timeout for recoverable object GETs.

Change-Id: I71f9824559126e4025e7629715ab9dac64231e09

doc/source/deployment_guide.rst

@@ -875,6 +875,10 @@ memcache_max_connections      2                Max number of connections to
                                                worker
 node_timeout                  10               Request timeout to external
                                                services
+recoverable_node_timeout      node_timeout     Request timeout to external
+                                               services for requests that, on
+                                               failure, can be recovered
+                                               from. For example, object GET.
 client_timeout                60               Timeout to read one chunk
                                                from a client
 conn_timeout                  0.5              Connection timeout to

etc/proxy-server.conf-sample

@@ -84,7 +84,20 @@ use = egg:swift#proxy
 # recheck_container_existence = 60
 # object_chunk_size = 8192
 # client_chunk_size = 8192
+#
+# How long the proxy server will wait on responses from the a/c/o servers.
 # node_timeout = 10
+#
+# How long the proxy server will wait for an initial response and to read a
+# chunk of data from the object servers while serving GET / HEAD requests.
+# Timeouts from these requests can be recovered from so setting this to
+# something lower than node_timeout would provide quicker error recovery
+# while allowing for a longer timeout for non-recoverable requests (PUTs).
+# Defaults to node_timeout, should be overriden if node_timeout is set to a
+# high number to prevent client timeouts from firing before the proxy server
+# has a chance to retry.
+# recoverable_node_timeout = node_timeout
+#
 # conn_timeout = 0.5
 #
 # How long to wait for requests to finish after a quorum has been established.

swift/proxy/controllers/base.py

@@ -654,9 +654,12 @@ class GetOrHeadHandler(object):
         try:
             nchunks = 0
             bytes_read_from_source = 0
+            node_timeout = self.app.node_timeout
+            if self.server_type == 'Object':
+                node_timeout = self.app.recoverable_node_timeout
             while True:
                 try:
-                    with ChunkReadTimeout(self.app.node_timeout):
+                    with ChunkReadTimeout(node_timeout):
                         chunk = source.read(self.app.object_chunk_size)
                         nchunks += 1
                         bytes_read_from_source += len(chunk)
@@ -724,13 +727,15 @@ class GetOrHeadHandler(object):
                 close_swift_conn(source)
 
     def _get_source_and_node(self):
-
         self.statuses = []
         self.reasons = []
         self.bodies = []
         self.source_headers = []
         sources = []
 
+        node_timeout = self.app.node_timeout
+        if self.server_type == 'Object' and not self.newest:
+            node_timeout = self.app.recoverable_node_timeout
         for node in self.app.iter_nodes(self.ring, self.partition):
             if node in self.used_nodes:
                 continue
@@ -744,7 +749,7 @@ class GetOrHeadHandler(object):
                         query_string=self.req_query_string)
                 self.app.set_node_timing(node, time.time() - start_node_timing)
 
-                with Timeout(self.app.node_timeout):
+                with Timeout(node_timeout):
                     possible_source = conn.getresponse()
                     # See NOTE: swift_conn at top of file about this.
                     possible_source.swift_conn = conn
@@ -1155,7 +1160,7 @@ class Controller(object):
         Base handler for HTTP GET or HEAD requests.
 
         :param req: swob.Request object
-        :param server_type: server type
+        :param server_type: server type used in logging
         :param ring: the ring to obtain nodes from
         :param partition: partition
         :param path: path for the request
@@ -1164,7 +1169,7 @@ class Controller(object):
         backend_headers = self.generate_request_headers(
             req, additional=req.headers)
 
-        handler = GetOrHeadHandler(self.app, req, server_type, ring,
+        handler = GetOrHeadHandler(self.app, req, self.server_type, ring,
                                    partition, path, backend_headers)
         res = handler.get_working_response(req)
 

swift/proxy/server.py

@@ -78,6 +78,8 @@ class Application(object):
 
         swift_dir = conf.get('swift_dir', '/etc/swift')
         self.node_timeout = int(conf.get('node_timeout', 10))
+        self.recoverable_node_timeout = int(
+            conf.get('recoverable_node_timeout', self.node_timeout))
         self.conn_timeout = float(conf.get('conn_timeout', 0.5))
         self.client_timeout = int(conf.get('client_timeout', 60))
         self.put_queue_depth = int(conf.get('put_queue_depth', 10))

test/unit/proxy/test_server.py

@@ -1697,7 +1697,7 @@ class TestObjectController(unittest.TestCase):
             except ChunkReadTimeout:
                 got_exc = True
             self.assert_(not got_exc)
-            self.app.node_timeout = 0.1
+            self.app.recoverable_node_timeout = 0.1
             set_http_connect(200, 200, 200, slow=True)
             resp = req.get_response(self.app)
             got_exc = False
@@ -1712,7 +1712,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
             self.app.update_request(req)
 
-            self.app.node_timeout = 0.1
+            self.app.recoverable_node_timeout = 0.1
             set_http_connect(200, 200, 200, slow=[3])
             resp = req.get_response(self.app)
             got_exc = False