Add config of server start timeouts for probetests

Currently the timeout for a wsgi server successfully binding to a port
and for a probetest background service to finish starting are hard coded
to 30 seconds.  While a reasonable default for most configurations, a
small virtualized environment may need a little more time in order for
probe tests to complete successfully.

This patch adds a 'bind_timeout' option to the DEFAULT section of the
main wsgi servers' config.  Also a new [probe_test] section and
'check_server_timeout' option to test.conf

DocImpact

Change-Id: Ibcaff153c7633bbf32e460fd9dbf04932eddb56f

doc/source/deployment_guide.rst

@@ -230,6 +230,7 @@ mount_check          true        Whether or not check if the devices are
                                  to the root device
 bind_ip              0.0.0.0     IP Address for server to bind to
 bind_port            6000        Port for server to bind to
+bind_timeout         30          Seconds to attempt bind before giving up
 workers              1           Number of workers to fork
 disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                  underlying filesystem does not support it.
@@ -338,6 +339,7 @@ mount_check          true        Whether or not check if the devices are
                                  to the root device
 bind_ip              0.0.0.0     IP Address for server to bind to
 bind_port            6001        Port for server to bind to
+bind_timeout         30          Seconds to attempt bind before giving up
 workers              1           Number of workers to fork
 user                 swift       User to run as
 disable_fallocate    false       Disable "fast fail" fallocate checks if the
@@ -436,6 +438,7 @@ mount_check          true        Whether or not check if the devices are
                                  to the root device
 bind_ip              0.0.0.0     IP Address for server to bind to
 bind_port            6002        Port for server to bind to
+bind_timeout         30          Seconds to attempt bind before giving up
 workers              1           Number of workers to fork
 user                 swift       User to run as
 db_preallocation     off         If you don't mind the extra disk space usage in
@@ -526,6 +529,8 @@ Option                        Default          Description
 bind_ip                       0.0.0.0          IP Address for server to
                                                bind to
 bind_port                     80               Port for server to bind to
+bind_timeout                  30               Seconds to attempt bind before
+                                               giving up
 swift_dir                     /etc/swift       Swift configuration directory
 workers                       1                Number of workers to fork
 user                          swift            User to run as

etc/account-server.conf-sample

@@ -1,6 +1,7 @@
 [DEFAULT]
 # bind_ip = 0.0.0.0
 # bind_port = 6002
+# bind_timeout = 30
 # backlog = 4096
 # workers = 1
 # user = swift

etc/container-server.conf-sample

@@ -1,6 +1,7 @@
 [DEFAULT]
 # bind_ip = 0.0.0.0
 # bind_port = 6001
+# bind_timeout = 30
 # backlog = 4096
 # workers = 1
 # user = swift

etc/object-server.conf-sample

@@ -1,6 +1,7 @@
 [DEFAULT]
 # bind_ip = 0.0.0.0
 # bind_port = 6000
+# bind_timeout = 30
 # backlog = 4096
 # workers = 1
 # user = swift

etc/proxy-server.conf-sample

@@ -1,6 +1,7 @@
 [DEFAULT]
 # bind_ip = 0.0.0.0
 # bind_port = 80
+# bind_timeout = 30
 # backlog = 4096
 # swift_dir = /etc/swift
 # workers = 1

swift/common/wsgi.py

@@ -70,7 +70,8 @@ def get_socket(conf, default_port=8080):
         bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)
         if addr[0] in (socket.AF_INET, socket.AF_INET6)][0]
     sock = None
-    retry_until = time.time() + 30
+    bind_timeout = int(conf.get('bind_timeout', 30))
+    retry_until = time.time() + bind_timeout
     warn_ssl = False
     while not sock and time.time() < retry_until:
         try:
@@ -85,8 +86,9 @@ def get_socket(conf, default_port=8080):
                 raise
             sleep(0.1)
     if not sock:
-        raise Exception('Could not bind to %s:%s after trying for 30 seconds' %
+        raise Exception(_('Could not bind to %s:%s '
-                        bind_addr)
+                          'after trying for %s seconds') % (
+                              bind_addr[0], bind_addr[1], bind_timeout))
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     # in my experience, sockets can hang around forever without keepalive
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

test/probe/__init__.py

@@ -0,0 +1,3 @@
+from test import get_config
+config = get_config('probe_test')
+CHECK_SERVER_TIMEOUT = int(config.get('check_server_timeout', 30))

test/probe/common.py

@@ -23,6 +23,8 @@ from swiftclient import get_auth, head_account
 
 from swift.common.ring import Ring
 
+from test.probe import CHECK_SERVER_TIMEOUT
+
 
 def start_server(port, port2server, pids, check=True):
     server = port2server[port]
@@ -40,7 +42,7 @@ def start_server(port, port2server, pids, check=True):
     return None
 
 
-def check_server(port, port2server, pids):
+def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
     server = port2server[port]
     if server[:-1] in ('account', 'container', 'object'):
         path = '/connect/1/2'
@@ -48,7 +50,7 @@ def check_server(port, port2server, pids):
             path += '/3'
         elif server[:-1] == 'object':
             path += '/3/4'
-        try_until = time() + 30
+        try_until = time() + timeout
         while True:
             try:
                 conn = HTTPConnection('127.0.0.1', port)
@@ -61,12 +63,12 @@ def check_server(port, port2server, pids):
             except Exception, err:
                 if time() > try_until:
                     print err
-                    print 'Giving up on %s:%s after 30 seconds.' % (
+                    print 'Giving up on %s:%s after %s seconds.' % (
-                        server, port)
+                        server, port, timeout)
                     raise err
                 sleep(0.1)
     else:
-        try_until = time() + 30
+        try_until = time() + timeout
         while True:
             try:
                 url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',

test/sample.conf

@@ -38,3 +38,6 @@ collate = C
 
 [unit_test]
 fake_syslog = False
+
+[probe_test]
+# check_server_timeout = 30