Check swift.conf MD5 with recon

I've seen several folks recently have problems with their Swift
clusters because they had different hash prefixes on different
nodes. Let's help them out by having recon check that.

Note that MD5-equality is stronger than what we need (which is
ConfigParser-equality for a particular set of keys), but this way we
don't expose the secret hash prefix and suffix across the internal
network, just the MD5 checksum of the file containing them.

Change-Id: I3af984ee45947345891b3c596a88e3464f178cc7
This commit is contained in:
Samuel Merritt 2014-04-08 11:27:14 -07:00
parent 4721deeb40
commit 31dac18625
4 changed files with 246 additions and 105 deletions
swift
cli
common/middleware
test/unit
cli
common/middleware

@ -16,8 +16,10 @@
cmdline utility to perform cluster reconnaissance
"""
from __future__ import print_function
from eventlet.green import urllib2
from swift.common.utils import SWIFT_CONF_FILE
from swift.common.ring import Ring
from urlparse import urlparse
try:
@ -82,16 +84,16 @@ class Scout(object):
body = urllib2.urlopen(url, timeout=self.timeout).read()
content = json.loads(body)
if self.verbose:
print "-> %s: %s" % (url, content)
print("-> %s: %s" % (url, content))
status = 200
except urllib2.HTTPError as err:
if not self.suppress_errors or self.verbose:
print "-> %s: %s" % (url, err)
print("-> %s: %s" % (url, err))
content = err
status = err.code
except urllib2.URLError as err:
if not self.suppress_errors or self.verbose:
print "-> %s: %s" % (url, err)
print("-> %s: %s" % (url, err))
content = err
status = -1
return url, content, status
@ -143,10 +145,10 @@ class SwiftRecon(object):
:param stats: dict of stats generated by _gen_stats
"""
print '[%(name)s] low: %(low)d, high: %(high)d, avg: ' \
'%(average).1f, total: %(total)d, ' \
'Failed: %(perc_none).1f%%, no_result: %(number_none)d, ' \
'reported: %(reported)d' % stats
print('[%(name)s] low: %(low)d, high: %(high)d, avg: '
'%(average).1f, total: %(total)d, '
'Failed: %(perc_none).1f%%, no_result: %(number_none)d, '
'reported: %(reported)d' % stats)
def _ptime(self, timev=None):
"""
@ -158,6 +160,21 @@ class SwiftRecon(object):
else:
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
def _md5_file(self, path):
"""
Get the MD5 checksum of a file.
:param path: path to file
:returns: MD5 checksum, hex encoded
"""
md5sum = md5()
with open(path, 'rb') as f:
block = f.read(4096)
while block:
md5sum.update(block)
block = f.read(4096)
return md5sum.hexdigest()
def get_devices(self, zone_filter, swift_dir, ring_name):
"""
Get a list of hosts in the ring
@ -183,36 +200,59 @@ class SwiftRecon(object):
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
:param ringfile: The local ring file to compare the md5sum with.
"""
stats = {}
matches = 0
errors = 0
md5sum = md5()
with open(ringfile, 'rb') as f:
block = f.read(4096)
while block:
md5sum.update(block)
block = f.read(4096)
ring_sum = md5sum.hexdigest()
ring_sum = self._md5_file(ringfile)
recon = Scout("ringmd5", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking ring md5sums" % self._ptime()
print("[%s] Checking ring md5sums" % self._ptime())
if self.verbose:
print "-> On disk %s md5sum: %s" % (ringfile, ring_sum)
print("-> On disk %s md5sum: %s" % (ringfile, ring_sum))
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
stats[url] = response[ringfile]
if response[ringfile] != ring_sum:
print "!! %s (%s) doesn't match on disk md5sum" % \
(url, response[ringfile])
print("!! %s (%s) doesn't match on disk md5sum" %
(url, response[ringfile]))
else:
matches = matches + 1
if self.verbose:
print "-> %s matches." % url
print("-> %s matches." % url)
else:
errors = errors + 1
print "%s/%s hosts matched, %s error[s] while checking hosts." \
% (matches, len(hosts), errors)
print "=" * 79
print("%s/%s hosts matched, %s error[s] while checking hosts."
% (matches, len(hosts), errors))
print("=" * 79)
def get_swiftconfmd5(self, hosts, printfn=print):
"""
Compare swift.conf md5sum with that on remote hosts
:param hosts: set of hosts to check. in the format of:
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
:param printfn: function to print text; defaults to print()
"""
matches = 0
errors = 0
conf_sum = self._md5_file(SWIFT_CONF_FILE)
recon = Scout("swiftconfmd5", self.verbose, self.suppress_errors,
self.timeout)
printfn("[%s] Checking swift.conf md5sum" % self._ptime())
if self.verbose:
printfn("-> On disk swift.conf md5sum: %s" % (conf_sum,))
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
if response[SWIFT_CONF_FILE] != conf_sum:
printfn("!! %s (%s) doesn't match on disk md5sum" %
(url, response[SWIFT_CONF_FILE]))
else:
matches = matches + 1
if self.verbose:
printfn("-> %s matches." % url)
else:
errors = errors + 1
printfn("%s/%s hosts matched, %s error[s] while checking hosts."
% (matches, len(hosts), errors))
printfn("=" * 79)
def async_check(self, hosts):
"""
@ -224,7 +264,7 @@ class SwiftRecon(object):
scan = {}
recon = Scout("async", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking async pendings" % self._ptime()
print("[%s] Checking async pendings" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
scan[url] = response['async_pending']
@ -232,8 +272,8 @@ class SwiftRecon(object):
if stats['reported'] > 0:
self._print_stats(stats)
else:
print "[async_pending] - No hosts returned valid data."
print "=" * 79
print("[async_pending] - No hosts returned valid data.")
print("=" * 79)
def umount_check(self, hosts):
"""
@ -246,8 +286,8 @@ class SwiftRecon(object):
errors = {}
recon = Scout("unmounted", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Getting unmounted drives from %s hosts..." % \
(self._ptime(), len(hosts))
print("[%s] Getting unmounted drives from %s hosts..." %
(self._ptime(), len(hosts)))
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
unmounted[url] = []
@ -260,12 +300,12 @@ class SwiftRecon(object):
for host in unmounted:
node = urlparse(host).netloc
for entry in unmounted[host]:
print "Not mounted: %s on %s" % (entry, node)
print("Not mounted: %s on %s" % (entry, node))
for host in errors:
node = urlparse(host).netloc
for entry in errors[host]:
print "Device errors: %s on %s" % (entry, node)
print "=" * 79
print("Device errors: %s on %s" % (entry, node))
print("=" * 79)
def expirer_check(self, hosts):
"""
@ -277,7 +317,7 @@ class SwiftRecon(object):
stats = {'object_expiration_pass': [], 'expired_last_pass': []}
recon = Scout("expirer/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout)
print "[%s] Checking on expirers" % self._ptime()
print("[%s] Checking on expirers" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
stats['object_expiration_pass'].append(
@ -290,10 +330,10 @@ class SwiftRecon(object):
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[%s] - No hosts returned valid data." % k
print("[%s] - No hosts returned valid data." % k)
else:
print "[%s] - No hosts returned valid data." % k
print "=" * 79
print("[%s] - No hosts returned valid data." % k)
print("=" * 79)
def replication_check(self, hosts):
"""
@ -306,7 +346,7 @@ class SwiftRecon(object):
'attempted': []}
recon = Scout("replication/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout)
print "[%s] Checking on replication" % self._ptime()
print("[%s] Checking on replication" % self._ptime())
least_recent_time = 9999999999
least_recent_url = None
most_recent_time = 0
@ -336,29 +376,29 @@ class SwiftRecon(object):
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[%s] - No hosts returned valid data." % k
print("[%s] - No hosts returned valid data." % k)
else:
print "[%s] - No hosts returned valid data." % k
print("[%s] - No hosts returned valid data." % k)
if least_recent_url is not None:
host = urlparse(least_recent_url).netloc
if not least_recent_time:
print 'Oldest completion was NEVER by %s.' % host
print('Oldest completion was NEVER by %s.' % host)
else:
elapsed = time.time() - least_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Oldest completion was %s (%d %s ago) by %s.' % (
print('Oldest completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(least_recent_time)),
elapsed, elapsed_unit, host)
elapsed, elapsed_unit, host))
if most_recent_url is not None:
host = urlparse(most_recent_url).netloc
elapsed = time.time() - most_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Most recent completion was %s (%d %s ago) by %s.' % (
print('Most recent completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(most_recent_time)),
elapsed, elapsed_unit, host)
print "=" * 79
elapsed, elapsed_unit, host))
print("=" * 79)
def object_replication_check(self, hosts):
"""
@ -370,7 +410,7 @@ class SwiftRecon(object):
stats = {}
recon = Scout("replication", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking on replication" % self._ptime()
print("[%s] Checking on replication" % self._ptime())
least_recent_time = 9999999999
least_recent_url = None
most_recent_time = 0
@ -391,29 +431,29 @@ class SwiftRecon(object):
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[replication_time] - No hosts returned valid data."
print("[replication_time] - No hosts returned valid data.")
else:
print "[replication_time] - No hosts returned valid data."
print("[replication_time] - No hosts returned valid data.")
if least_recent_url is not None:
host = urlparse(least_recent_url).netloc
if not least_recent_time:
print 'Oldest completion was NEVER by %s.' % host
print('Oldest completion was NEVER by %s.' % host)
else:
elapsed = time.time() - least_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Oldest completion was %s (%d %s ago) by %s.' % (
print('Oldest completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(least_recent_time)),
elapsed, elapsed_unit, host)
elapsed, elapsed_unit, host))
if most_recent_url is not None:
host = urlparse(most_recent_url).netloc
elapsed = time.time() - most_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Most recent completion was %s (%d %s ago) by %s.' % (
print('Most recent completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(most_recent_time)),
elapsed, elapsed_unit, host)
print "=" * 79
elapsed, elapsed_unit, host))
print("=" * 79)
def updater_check(self, hosts):
"""
@ -425,7 +465,7 @@ class SwiftRecon(object):
stats = []
recon = Scout("updater/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout)
print "[%s] Checking updater times" % self._ptime()
print("[%s] Checking updater times" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
if response['%s_updater_sweep' % self.server_type]:
@ -436,10 +476,10 @@ class SwiftRecon(object):
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[updater_last_sweep] - No hosts returned valid data."
print("[updater_last_sweep] - No hosts returned valid data.")
else:
print "[updater_last_sweep] - No hosts returned valid data."
print "=" * 79
print("[updater_last_sweep] - No hosts returned valid data.")
print("=" * 79)
def auditor_check(self, hosts):
"""
@ -455,12 +495,12 @@ class SwiftRecon(object):
asince = '%s_audits_since' % self.server_type
recon = Scout("auditor/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout)
print "[%s] Checking auditor stats" % self._ptime()
print("[%s] Checking auditor stats" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
scan[url] = response
if len(scan) < 1:
print "Error: No hosts available"
print("Error: No hosts available")
return
stats = {}
stats[adone] = [scan[i][adone] for i in scan
@ -473,7 +513,7 @@ class SwiftRecon(object):
if scan[i][asince] is not None]
for k in stats:
if len(stats[k]) < 1:
print "[%s] - No hosts returned valid data." % k
print("[%s] - No hosts returned valid data." % k)
else:
if k != asince:
computed = self._gen_stats(stats[k], k)
@ -484,9 +524,9 @@ class SwiftRecon(object):
high = max(stats[asince])
total = sum(stats[asince])
average = total / len(stats[asince])
print '[last_pass] oldest: %s, newest: %s, avg: %s' % \
(self._ptime(low), self._ptime(high), self._ptime(average))
print "=" * 79
print('[last_pass] oldest: %s, newest: %s, avg: %s' %
(self._ptime(low), self._ptime(high), self._ptime(average)))
print("=" * 79)
def nested_get_value(self, key, recon_entry):
"""
@ -522,7 +562,7 @@ class SwiftRecon(object):
quarantined = 'quarantined'
recon = Scout("auditor/object", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking auditor stats " % self._ptime()
print("[%s] Checking auditor stats " % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
if response['object_auditor_stats_ALL']:
@ -545,16 +585,16 @@ class SwiftRecon(object):
if None in stats[k]:
stats[k] = [x for x in stats[k] if x is not None]
if len(stats[k]) < 1:
print "[Auditor %s] - No hosts returned valid data." % k
print("[Auditor %s] - No hosts returned valid data." % k)
else:
computed = self._gen_stats(stats[k],
name='ALL_%s_last_path' % k)
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[ALL_auditor] - No hosts returned valid data."
print("[ALL_auditor] - No hosts returned valid data.")
else:
print "[ALL_auditor] - No hosts returned valid data."
print("[ALL_auditor] - No hosts returned valid data.")
if len(zbf_scan) > 0:
stats = {}
stats[atime] = [(self.nested_get_value(atime, zbf_scan[i]))
@ -569,17 +609,17 @@ class SwiftRecon(object):
if None in stats[k]:
stats[k] = [x for x in stats[k] if x is not None]
if len(stats[k]) < 1:
print "[Auditor %s] - No hosts returned valid data." % k
print("[Auditor %s] - No hosts returned valid data." % k)
else:
computed = self._gen_stats(stats[k],
name='ZBF_%s_last_path' % k)
if computed['reported'] > 0:
self._print_stats(computed)
else:
print "[ZBF_auditor] - No hosts returned valid data."
print("[ZBF_auditor] - No hosts returned valid data.")
else:
print "[ZBF_auditor] - No hosts returned valid data."
print "=" * 79
print("[ZBF_auditor] - No hosts returned valid data.")
print("=" * 79)
def load_check(self, hosts):
"""
@ -593,7 +633,7 @@ class SwiftRecon(object):
load15 = {}
recon = Scout("load", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking load averages" % self._ptime()
print("[%s] Checking load averages" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
load1[url] = response['1m']
@ -606,8 +646,8 @@ class SwiftRecon(object):
name='%s_load_avg' % item)
self._print_stats(computed)
else:
print "[%s_load_avg] - No hosts returned valid data." % item
print "=" * 79
print("[%s_load_avg] - No hosts returned valid data." % item)
print("=" * 79)
def quarantine_check(self, hosts):
"""
@ -621,7 +661,7 @@ class SwiftRecon(object):
acctq = {}
recon = Scout("quarantined", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking quarantine" % self._ptime()
print("[%s] Checking quarantine" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
objq[url] = response['objects']
@ -634,8 +674,8 @@ class SwiftRecon(object):
name='quarantined_%s' % item)
self._print_stats(computed)
else:
print "No hosts returned valid data."
print "=" * 79
print("No hosts returned valid data.")
print("=" * 79)
def socket_usage(self, hosts):
"""
@ -651,7 +691,7 @@ class SwiftRecon(object):
orphan = {}
recon = Scout("sockstat", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking socket usage" % self._ptime()
print("[%s] Checking socket usage" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
inuse4[url] = response['tcp_in_use']
@ -667,8 +707,8 @@ class SwiftRecon(object):
computed = self._gen_stats(stats[item].values(), item)
self._print_stats(computed)
else:
print "No hosts returned valid data."
print "=" * 79
print("No hosts returned valid data.")
print("=" * 79)
def disk_usage(self, hosts, top=0, human_readable=False):
"""
@ -686,14 +726,14 @@ class SwiftRecon(object):
top_percents = [(None, 0)] * top
recon = Scout("diskusage", self.verbose, self.suppress_errors,
self.timeout)
print "[%s] Checking disk usage now" % self._ptime()
print("[%s] Checking disk usage now" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
hostusage = []
for entry in response:
if not isinstance(entry['mounted'], bool):
print "-> %s/%s: Error: %s" % (url, entry['device'],
entry['mounted'])
print("-> %s/%s: Error: %s" % (url, entry['device'],
entry['mounted']))
elif entry['mounted']:
used = float(entry['used']) / float(entry['size']) \
* 100.0
@ -719,17 +759,17 @@ class SwiftRecon(object):
for percent in stats[url]:
percents[int(percent)] = percents.get(int(percent), 0) + 1
else:
print "-> %s: Error. No drive info available." % url
print("-> %s: Error. No drive info available." % url)
if len(lows) > 0:
low = min(lows)
high = max(highs)
# dist graph shamelessly stolen from https://github.com/gholt/tcod
print "Distribution Graph:"
print("Distribution Graph:")
mul = 69.0 / max(percents.values())
for percent in sorted(percents):
print '% 3d%%%5d %s' % (percent, percents[percent],
'*' * int(percents[percent] * mul))
print('% 3d%%%5d %s' % (percent, percents[percent],
'*' * int(percents[percent] * mul)))
raw_used = sum(raw_total_used)
raw_avail = sum(raw_total_avail)
raw_total = raw_used + raw_avail
@ -738,26 +778,26 @@ class SwiftRecon(object):
raw_used = size_suffix(raw_used)
raw_avail = size_suffix(raw_avail)
raw_total = size_suffix(raw_total)
print "Disk usage: space used: %s of %s" % (raw_used, raw_total)
print "Disk usage: space free: %s of %s" % (raw_avail, raw_total)
print "Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" % \
(low, high, avg_used)
print("Disk usage: space used: %s of %s" % (raw_used, raw_total))
print("Disk usage: space free: %s of %s" % (raw_avail, raw_total))
print("Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" %
(low, high, avg_used))
else:
print "No hosts returned valid data."
print "=" * 79
print("No hosts returned valid data.")
print("=" * 79)
if top_percents:
print 'TOP %s' % top
print('TOP %s' % top)
for ident, used in top_percents:
if ident:
url, device = ident.split()
host = urlparse(url).netloc.split(':')[0]
print '%.02f%% %s' % (used, '%-15s %s' % (host, device))
print('%.02f%% %s' % (used, '%-15s %s' % (host, device)))
def main(self):
"""
Retrieve and report cluster info from hosts running recon middleware.
"""
print "=" * 79
print("=" * 79)
usage = '''
usage: %prog <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d]
[-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
@ -820,7 +860,7 @@ class SwiftRecon(object):
if arguments[0] in self.check_types:
self.server_type = arguments[0]
else:
print "Invalid Server Type"
print("Invalid Server Type")
args.print_help()
sys.exit(1)
else:
@ -837,8 +877,8 @@ class SwiftRecon(object):
else:
hosts = self.get_devices(None, swift_dir, self.server_type)
print "--> Starting reconnaissance on %s hosts" % len(hosts)
print "=" * 79
print("--> Starting reconnaissance on %s hosts" % len(hosts))
print("=" * 79)
if options.all:
if self.server_type == 'object':
@ -865,7 +905,7 @@ class SwiftRecon(object):
if self.server_type == 'object':
self.async_check(hosts)
else:
print "Error: Can't check asyncs on non object servers."
print("Error: Can't check asyncs on non object servers.")
if options.unmounted:
self.umount_check(hosts)
if options.replication:
@ -880,20 +920,21 @@ class SwiftRecon(object):
self.auditor_check(hosts)
if options.updater:
if self.server_type == 'account':
print "Error: Can't check updaters on account servers."
print("Error: Can't check updaters on account servers.")
else:
self.updater_check(hosts)
if options.expirer:
if self.server_type == 'object':
self.expirer_check(hosts)
else:
print "Error: Can't check expired on non object servers."
print("Error: Can't check expired on non object servers.")
if options.loadstats:
self.load_check(hosts)
if options.diskusage:
self.disk_usage(hosts, options.top, options.human_readable)
if options.md5:
self.get_ringmd5(hosts, ring_file)
self.get_swiftconfmd5(hosts)
if options.quarantined:
self.quarantine_check(hosts)
if options.sockstat:
@ -905,7 +946,7 @@ def main():
reconnoiter = SwiftRecon()
reconnoiter.main()
except KeyboardInterrupt:
print '\n'
print('\n')
if __name__ == '__main__':

@ -19,7 +19,8 @@ from swift import gettext_ as _
from swift import __version__ as swiftver
from swift.common.swob import Request, Response
from swift.common.utils import get_logger, config_true_value, json
from swift.common.utils import get_logger, config_true_value, json, \
SWIFT_CONF_FILE
from swift.common.constraints import check_mount
from resource import getpagesize
from hashlib import md5
@ -244,6 +245,23 @@ class ReconMiddleware(object):
self.logger.exception(_('Error reading ringfile'))
return sums
def get_swift_conf_md5(self, openr=open):
"""get md5 of swift.conf"""
md5sum = md5()
try:
with openr(SWIFT_CONF_FILE, 'r') as fh:
chunk = fh.read(4096)
while chunk:
md5sum.update(chunk)
chunk = fh.read(4096)
except IOError as err:
if err.errno != errno.ENOENT:
self.logger.exception(_('Error reading swift.conf'))
hexsum = None
else:
hexsum = md5sum.hexdigest()
return {SWIFT_CONF_FILE: hexsum}
def get_quarantine_count(self):
"""get obj/container/account quarantine counts"""
qcounts = {"objects": 0, "containers": 0, "accounts": 0}
@ -318,6 +336,8 @@ class ReconMiddleware(object):
content = self.get_diskusage()
elif rcheck == "ringmd5":
content = self.get_ring_md5()
elif rcheck == "swiftconfmd5":
content = self.get_swift_conf_md5()
elif rcheck == "quarantined":
content = self.get_quarantine_count()
elif rcheck == "sockstat":

@ -21,6 +21,7 @@ import string
import tempfile
import time
import unittest
import urlparse
from eventlet.green import urllib2
@ -146,3 +147,71 @@ class TestRecon(unittest.TestCase):
ips = self.recon_instance.get_devices(
1, self.swift_dir, self.ring_name)
self.assertEqual(set([('127.0.0.1', 10001)]), ips)
class TestReconCommands(unittest.TestCase):
def setUp(self):
self.recon = recon.SwiftRecon()
self.hosts = set([('127.0.0.1', 10000)])
def mock_responses(self, resps):
def fake_urlopen(url, timeout):
scheme, netloc, path, _, _, _ = urlparse.urlparse(url)
self.assertEqual(scheme, 'http') # can't handle anything else
self.assertTrue(path.startswith('/recon/'))
if ':' in netloc:
host, port = netloc.split(':', 1)
port = int(port)
else:
host = netloc
port = 80
response_body = resps[(host, port, path[7:])]
resp = mock.MagicMock()
resp.read = mock.MagicMock(side_effect=[response_body])
return resp
return mock.patch('eventlet.green.urllib2.urlopen', fake_urlopen)
def test_get_swiftconfmd5(self):
hosts = set([('10.1.1.1', 10000),
('10.2.2.2', 10000)])
cksum = '729cf900f2876dead617d088ece7fe8c'
responses = {
('10.1.1.1', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum}),
('10.2.2.2', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum})}
printed = []
with self.mock_responses(responses):
with mock.patch.object(self.recon, '_md5_file', lambda _: cksum):
self.recon.get_swiftconfmd5(hosts, printfn=printed.append)
output = '\n'.join(printed) + '\n'
self.assertTrue("2/2 hosts matched" in output)
def test_get_swiftconfmd5_mismatch(self):
hosts = set([('10.1.1.1', 10000),
('10.2.2.2', 10000)])
cksum = '29d5912b1fcfcc1066a7f51412769c1d'
responses = {
('10.1.1.1', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum}),
('10.2.2.2', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': 'bogus'})}
printed = []
with self.mock_responses(responses):
with mock.patch.object(self.recon, '_md5_file', lambda _: cksum):
self.recon.get_swiftconfmd5(hosts, printfn=printed.append)
output = '\n'.join(printed) + '\n'
self.assertTrue("1/2 hosts matched" in output)
self.assertTrue("http://10.2.2.2:10000/recon/swiftconfmd5 (bogus) "
"doesn't match on disk md5sum" in output)

@ -161,6 +161,9 @@ class FakeRecon(object):
def fake_ringmd5(self):
return {'ringmd5test': "1"}
def fake_swiftconfmd5(self):
return {'/etc/swift/swift.conf': "abcdef"}
def fake_quarantined(self):
return {'quarantinedtest': "1"}
@ -725,6 +728,7 @@ class TestReconMiddleware(unittest.TestCase):
self.app.get_unmounted = self.frecon.fake_unmounted
self.app.get_diskusage = self.frecon.fake_diskusage
self.app.get_ring_md5 = self.frecon.fake_ringmd5
self.app.get_swift_conf_md5 = self.frecon.fake_swiftconfmd5
self.app.get_quarantine_count = self.frecon.fake_quarantined
self.app.get_socket_info = self.frecon.fake_sockstat
@ -916,6 +920,13 @@ class TestReconMiddleware(unittest.TestCase):
resp = self.app(req.environ, start_response)
self.assertEquals(resp, get_ringmd5_resp)
def test_recon_get_swiftconfmd5(self):
get_swiftconfmd5_resp = ['{"/etc/swift/swift.conf": "abcdef"}']
req = Request.blank('/recon/swiftconfmd5',
environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, get_swiftconfmd5_resp)
def test_recon_get_quarantined(self):
get_quarantined_resp = ['{"quarantinedtest": "1"}']
req = Request.blank('/recon/quarantined',