Add option to make probetests more brittle

Currently probetests take advantage of a number of assumptions about the SUT.
Unfortunately after some time with a working SAIO, configuration drift may
result in a system that is no longer compatible with these assumptions.  To
help weary developers more quickly identify the changes they've made since
they last ran probetests successfully, some handy validators have been added
to test.probe.common

Additionally a new option 'validate_rsync' in test.conf, when enabled, will
run a series of up front validations during the setup of each probetest by
inspecting the ring, the mounted devices, and the rsync exports ("modules") in
order to ensure that when probetests fail the do so early and with specific
complaints.

To preserve existing failures, the option is disabled by default.

Change-Id: I2be11c7e67ccd0bc0589c360c170049b6288c152

test/probe/__init__.py

@@ -1,3 +1,5 @@
 from test import get_config
+from swift.common.utils import config_true_value
 config = get_config('probe_test')
 CHECK_SERVER_TIMEOUT = int(config.get('check_server_timeout', 30))
+VALIDATE_RSYNC = config_true_value(config.get('validate_rsync', False))

test/probe/common.py

@@ -14,16 +14,19 @@
 # limitations under the License.
 
 from httplib import HTTPConnection
+import os
 from os import kill, path
 from signal import SIGTERM
 from subprocess import Popen, PIPE
+import sys
 from time import sleep, time
 
 from swiftclient import get_auth, head_account
 
 from swift.common.ring import Ring
+from swift.common.utils import readconf
 
-from test.probe import CHECK_SERVER_TIMEOUT
+from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
 
 
 def start_server(port, port2server, pids, check=True):
@@ -130,12 +133,61 @@ def kill_nonprimary_server(primary_nodes, port2server, pids):
             return port
 
 
+def get_ring(server, force_validate=None):
+    ring = Ring('/etc/swift/%s.ring.gz' % server)
+    if not VALIDATE_RSYNC and not force_validate:
+        return ring
+    # easy sanity checks
+    assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
+        ring.serialized_path, ring.replica_count)
+    assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
+        ring.serialized_path, len(ring.devs))
+    # map server to config by port
+    port_to_config = {}
+    for node_id in range(1,5):
+        conf = readconf('/etc/swift/%s-server/%d.conf' % (server, node_id),
+                        section_name='%s-replicator' % server)
+        port_to_config[int(conf['bind_port'])] = conf
+    for dev in ring.devs:
+        # verify server is exposing mounted device
+        conf = port_to_config[dev['port']]
+        for device in os.listdir(conf['devices']):
+            if device == dev['device']:
+                full_path = path.realpath(path.join(conf['devices'], device))
+                assert path.ismount(full_path), 'device %s in %s was not ' \
+                        'mounted (%s)' % (device, conf['devices'], full_path)
+                break
+        else:
+            assert False, "unable to find ring device %s " \
+                    "under %s's devices (%s)" % (
+                        dev['device'], server, conf['devices'])
+        # verify server is exposing rsync device
+        rsync_export = '%s%s' % (server, dev['replication_port'])
+        cmd = "rsync rsync://localhost/%s" % rsync_export
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        stdout, _stderr = p.communicate()
+        if p.returncode:
+            raise AssertionError('unable to connect to rsync '
+                                 'export %s (%s)' % (rsync_export, cmd))
+        for line in stdout.splitlines():
+            if line.rsplit(None, 1)[-1] == dev['device']:
+                break
+        else:
+            assert False, "unable to find ring device %s under rsync's " \
+                    "exported devices for %s (%s)" % (
+                        dev['device'], rsync_export, cmd)
+    return ring
+
+
 def reset_environment():
     p = Popen("resetswift 2>&1", shell=True, stdout=PIPE)
     stdout, _stderr = p.communicate()
     print stdout
     pids = {}
     try:
+        account_ring = get_ring('account')
+        container_ring = get_ring('container')
+        object_ring = get_ring('object')
         port2server = {}
         config_dict = {}
         for server, port in [('account', 6002), ('container', 6001),
@@ -148,15 +200,15 @@ def reset_environment():
             check_server(port, port2server, pids)
         port2server[8080] = 'proxy'
         url, token, account = start_server(8080, port2server, pids)
-        account_ring = Ring('/etc/swift/account.ring.gz')
-        container_ring = Ring('/etc/swift/container.ring.gz')
-        object_ring = Ring('/etc/swift/object.ring.gz')
         for name in ('account', 'container', 'object'):
             for server in (name, '%s-replicator' % name):
                 config_dict[server] = '/etc/swift/%s-server/%%d.conf' % name
     except BaseException:
         try:
             raise
+        except AssertionError, e:
+            print >>sys.stderr, 'ERROR: %s' % e
+            os._exit(1)
         finally:
             try:
                 kill_servers(port2server, pids)
@@ -202,3 +254,9 @@ def get_to_final_state():
                 'once']))
     for process in processes:
         process.wait()
+
+
+if __name__ == "__main__":
+    for server in ('account', 'container', 'object'):
+        get_ring(server, force_validate=True)
+        print '%s OK' % server

test/sample.conf

@@ -47,3 +47,4 @@ fake_syslog = False
 
 [probe_test]
 # check_server_timeout = 30
+# validate_rsync = false