Update bin scripts to be storage policy aware

swift-container-info:
    Print policy container info

swift-object-info:
    Allow to specify storage policy name when looking for object info
    Notify if there is missmatch between ring location and the actual
    object path in filesystem

swift-get-nodes:
    Allow to specify storage policy name when looking for account/
    container/object ring location
    Notify if there is missmatch between ring and the policy

Lookup policy name in swift.conf; 'Legacy' container will use
policy-0's name; 'Unknown' is shown if policy not found in swift.conf

DocImpact
Implements: blueprint storage-policies
Change-Id: I450d40dc6e2d8f759187dff36d658e52737ae2a5
This commit is contained in:
Yuan Zhou 2014-05-27 17:25:15 -07:00 committed by Clay Gerrard
parent c11ac01252
commit 6cc10d17de
4 changed files with 700 additions and 276 deletions

View File

@ -14,129 +14,70 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import optparse
import sys
import urllib
import os
from optparse import OptionParser
from swift.common.ring import Ring
from swift.common.utils import hash_path, storage_directory
from swift.cli.info import print_item_locations, InfoSystemExit
parser = optparse.OptionParser()
parser.add_option('-a', '--all', action='store_true',
if __name__ == '__main__':
usage = '''
Shows the nodes responsible for the item specified.
Usage: %prog [-a] <ring.gz> <account> [<container>] [<object>]
Or: %prog [-a] <ring.gz> -p partition
Or: %prog [-a] -P policy_name <account> <container> <object>
Note: account, container, object can also be a single arg separated by /
Example:
$ %prog -a /etc/swift/account.ring.gz MyAccount
Partition 5743883
Hash 96ae332a60b58910784e4417a03e1ad0
10.1.1.7:8000 sdd1
10.1.9.2:8000 sdb1
10.1.5.5:8000 sdf1
10.1.5.9:8000 sdt1 # [Handoff]
'''
parser = OptionParser(usage)
parser.add_option('-a', '--all', action='store_true',
help='Show all handoff nodes')
parser.add_option('-p', '--partition', metavar='PARTITION',
parser.add_option('-p', '--partition', metavar='PARTITION',
help='Show nodes for a given partition')
(options, args) = parser.parse_args()
parser.add_option('-P', '--policy-name', dest='policy_name',
help='Specify which policy to use')
parser.add_option('-d', '--swift-dir', default='/etc/swift',
dest='swift_dir', help='Path to swift directory')
options, args = parser.parse_args()
if (len(args) < 2 or len(args) > 4) and \
(options.partition is None or not args):
print 'Usage: %s [-a] <ring.gz> <account> [<container>] [<object>]' \
% sys.argv[0]
print ' Or: %s [-a] <ring.gz> -p partition' % sys.argv[0]
print ' Note: account, container, object can also be a single arg ' \
'separated by /'
print 'Shows the nodes responsible for the item specified.'
print 'Example:'
print ' $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0]
print ' Partition 5743883'
print ' Hash 96ae332a60b58910784e4417a03e1ad0'
print ' 10.1.1.7:8000 sdd1'
print ' 10.1.9.2:8000 sdb1'
print ' 10.1.5.5:8000 sdf1'
print ' 10.1.5.9:8000 sdt1 # [Handoff]'
sys.exit(1)
# swift-get-nodes -P nada -p 1
if len(args) == 0:
if not options.policy_name or not options.partition:
sys.exit(parser.print_help())
elif len(args) > 4 or len(args) < 1:
sys.exit(parser.print_help())
if len(args) == 2 and '/' in args[1]:
# Parse single path arg, as noted in above help text.
# if len(args) == 1 and options.policy_name and '/' in args[0]:
if len(args) == 1 and not args[0].endswith('ring.gz'):
path = args[0].lstrip('/')
args = [p for p in path.split('/', 2) if p]
if len(args) == 2 and '/' in args[1]:
path = args[1].lstrip('/')
args = [args[0]] + [p for p in path.split('/', 2) if p]
ringloc = None
account = None
container = None
obj = None
ring = None
ring_name = None
if len(args) == 4:
# Account, Container and Object
ring_file, account, container, obj = args
ring = Ring(ring_file)
hash_str = hash_path(account, container, obj)
part, nodes = ring.get_nodes(account, container, obj)
target = "%s/%s/%s" % (account, container, obj)
loc = 'objects'
elif len(args) == 3:
# Account, Container
ring_file, account, container = args
ring = Ring(ring_file)
hash_str = hash_path(account, container)
part, nodes = ring.get_nodes(account, container)
target = "%s/%s" % (account, container)
loc = 'containers'
elif len(args) == 2:
# Account
ring_file, account = args
ring = Ring(ring_file)
hash_str = hash_path(account)
part, nodes = ring.get_nodes(account)
target = "%s" % (account)
loc = 'accounts'
elif len(args) == 1:
# Partition
ring_file = args[0]
ring = Ring(ring_file)
hash_str = None
part = int(options.partition)
nodes = ring.get_part_nodes(part)
target = ''
loc = ring_file.rsplit('/', 1)[-1].split('.', 1)[0]
if loc in ('account', 'container', 'object'):
loc += 's'
if len(args) >= 1 and args[0].endswith('ring.gz'):
if os.path.exists(args[0]):
ring_name = args[0].rsplit('/', 1)[-1].split('.', 1)[0]
ring = Ring(args[0])
else:
loc = '<type>'
print 'Ring file does not exist'
args.pop(0)
more_nodes = []
for more_node in ring.get_more_nodes(part):
more_nodes.append(more_node)
if not options.all and len(more_nodes) >= len(nodes):
break
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n' % obj
print '\nPartition\t%s' % part
print 'Hash \t%s\n' % hash_str
for node in nodes:
print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
node['device'])
for mnode in more_nodes:
print 'Server:Port Device\t%s:%s %s\t [Handoff]' \
% (mnode['ip'], mnode['port'], mnode['device'])
print "\n"
for node in nodes:
print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
for mnode in more_nodes:
print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s" # [Handoff]' \
% (mnode['ip'], mnode['port'], mnode['device'], part,
urllib.quote(target))
print "\n"
print 'Use your own device location of servers:'
print 'such as "export DEVICE=/srv/node"'
for node in nodes:
if hash_str:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/"' % (
node['ip'], node['device'], storage_directory(loc, part, hash_str))
else:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/"' % (
node['ip'], node['device'], loc, part)
for mnode in more_nodes:
if hash_str:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/" '\
'# [Handoff]' % (mnode['ip'], mnode['device'],
storage_directory(loc, part, hash_str))
else:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/" # [Handoff]' % (
mnode['ip'], mnode['device'], loc, part)
try:
print_item_locations(ring, ring_name, *args, **vars(options))
except InfoSystemExit:
sys.exit(1)

View File

@ -14,112 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from datetime import datetime
from hashlib import md5
from optparse import OptionParser
from swift.common.ring import Ring
from swift.obj.diskfile import read_metadata
from swift.common.utils import hash_path, storage_directory
def print_object_info(datafile, check_etag=True, swift_dir='/etc/swift'):
if not os.path.exists(datafile) or not datafile.endswith('.data'):
print "Data file doesn't exist"
sys.exit(1)
try:
ring = Ring(swift_dir, ring_name='object')
except Exception:
ring = None
fp = open(datafile, 'rb')
metadata = read_metadata(fp)
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = metadata.pop('X-Timestamp', '')
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
if path:
print 'Path: %s' % path
account, container, obj = path.split('/', 3)[1:]
print ' Account: %s' % account
print ' Container: %s' % container
print ' Object: %s' % obj
obj_hash = hash_path(account, container, obj)
print ' Object hash: %s' % obj_hash
else:
print 'Path: Not found in metadata'
if content_type:
print 'Content-Type: %s' % content_type
else:
print 'Content-Type: Not found in metadata'
if ts:
print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts)
else:
print 'Timestamp: Not found in metadata'
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print "Etag: %s doesn't match file hash of %s!" % (etag, h)
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print "Content-Length: %s doesn't match file length of %s" % (
length, file_len)
else:
print 'Content-Length: Not found in metadata'
print 'User Metadata: %s' % metadata
if ring is not None:
print 'Ring locations:'
part, nodes = ring.get_nodes(account, container, obj)
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.data' %
(node['ip'], node['port'], node['device'],
storage_directory('objects', part, obj_hash), ts))
print
print 'note: /srv/node is used as default value of `devices`, '\
'the real value is set in object-server.conf '\
'on each storage node.'
fp.close()
from swift.cli.info import print_obj, InfoSystemExit
if __name__ == '__main__':
parser = OptionParser()
parser.set_defaults(check_etag=True, swift_dir='/etc/swift')
parser = OptionParser('%prog [options] OBJECT_FILE')
parser.add_option(
'-n', '--no-check-etag',
'-n', '--no-check-etag', default=True,
action="store_false", dest="check_etag",
help="Don't verify file contents against stored etag")
parser.add_option(
'-d', '--swift-dir',
'-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
help="Pass location of swift directory")
parser.add_option(
'-P', '--policy-name', dest='policy_name',
help="Specify storage policy name")
options, args = parser.parse_args()
if len(args) < 1:
print "Usage: %s [-n] [-d] OBJECT_FILE" % sys.argv[0]
sys.exit(1)
if len(args) != 1:
sys.exit(parser.print_help())
print_object_info(args[0], check_etag=options.check_etag,
swift_dir=options.swift_dir)
try:
print_obj(*args, **vars(options))
except InfoSystemExit:
sys.exit(1)

View File

@ -10,9 +10,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import os
import sqlite3
import urllib
from datetime import datetime
from hashlib import md5
from swift.common.utils import hash_path, storage_directory
from swift.common.ring import Ring
@ -20,6 +23,9 @@ from swift.common.request_helpers import is_sys_meta, is_user_meta, \
strip_sys_meta_prefix, strip_user_meta_prefix
from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
extract_policy_index
from swift.common.storage_policy import POLICIES, POLICY_INDEX
class InfoSystemExit(Exception):
@ -29,35 +35,105 @@ class InfoSystemExit(Exception):
pass
def print_ring_locations(ring, datadir, account, container=None):
def print_ring_locations(ring, datadir, account, container=None, obj=None,
tpart=None, all_nodes=False, policy_index=None):
"""
print out ring locations of specified type
:param ring: ring instance
:param datadir: high level directory to store account/container/objects
:param datadir: name of directory where things are stored. Usually one of
"accounts", "containers", "objects", or "objects-N".
:param account: account name
:param container: container name
:param obj: object name
:param tpart: target partition in ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
:param policy_index: include policy_index in curl headers
"""
if ring is None or datadir is None or account is None:
raise ValueError('None type')
storage_type = 'account'
if container:
storage_type = 'container'
try:
part, nodes = ring.get_nodes(account, container, None)
except (ValueError, AttributeError):
raise ValueError('Ring error')
if not ring:
raise ValueError("No ring specified")
if not datadir:
raise ValueError("No datadir specified")
if tpart is None and not account:
raise ValueError("No partition or account/container/object specified")
if not account and (container or obj):
raise ValueError("Container/object specified without account")
if obj and not container:
raise ValueError('Object specified without container')
if obj:
target = '%s/%s/%s' % (account, container, obj)
elif container:
target = '%s/%s' % (account, container)
else:
path_hash = hash_path(account, container, None)
print '\nRing locations:'
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.db' %
(node['ip'], node['port'], node['device'],
storage_directory(datadir, part, path_hash),
path_hash))
print '\nnote: /srv/node is used as default value of `devices`, the ' \
'real value is set in the %s config file on each storage node.' % \
storage_type
target = '%s' % (account)
if tpart:
part = int(tpart)
else:
part = ring.get_part(account, container, obj)
primary_nodes = ring.get_part_nodes(part)
handoff_nodes = ring.get_more_nodes(part)
if not all_nodes:
handoff_nodes = itertools.islice(handoff_nodes, len(primary_nodes))
handoff_nodes = list(handoff_nodes)
if account and not tpart:
path_hash = hash_path(account, container, obj)
else:
path_hash = None
print 'Partition\t%s' % part
print 'Hash \t%s\n' % path_hash
for node in primary_nodes:
print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
node['device'])
for node in handoff_nodes:
print 'Server:Port Device\t%s:%s %s\t [Handoff]' % (
node['ip'], node['port'], node['device'])
print "\n"
for node in primary_nodes:
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
if policy_index is not None:
cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index)
print cmd
for node in handoff_nodes:
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
if policy_index is not None:
cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index)
cmd += ' # [Handoff]'
print cmd
print "\n\nUse your own device location of servers:"
print "such as \"export DEVICE=/srv/node\""
if path_hash:
for node in primary_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' %
(node['ip'], node['device'],
storage_directory(datadir, part, path_hash)))
for node in handoff_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' %
(node['ip'], node['device'],
storage_directory(datadir, part, path_hash)))
else:
for node in primary_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' %
(node['ip'], node['device'], datadir, part))
for node in handoff_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"'
' # [Handoff]' %
(node['ip'], node['device'], datadir, part))
print '\nnote: `/srv/node*` is used as default value of `devices`, the ' \
'real value is set in the config file on each storage node.'
def print_db_info_metadata(db_type, info, metadata):
@ -106,11 +182,20 @@ def print_db_info_metadata(db_type, info, metadata):
print (' Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['delete_timestamp'])),
info['delete_timestamp']))
print (' Status Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['status_changed_at'])),
info['status_changed_at']))
if db_type == 'account':
print ' Container Count: %s' % info['container_count']
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
if db_type == 'container':
try:
policy_name = POLICIES[info['storage_policy_index']].name
except KeyError:
policy_name = 'Unknown'
print (' Storage Policy: %s (%s)' % (
policy_name, info['storage_policy_index']))
print (' Reported Put Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(
float(info['reported_put_timestamp'])),
@ -123,8 +208,8 @@ def print_db_info_metadata(db_type, info, metadata):
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
print ' Chexor: %s' % info['hash']
print ' UUID: %s' % info['id']
except KeyError:
raise ValueError('Info is incomplete')
except KeyError as e:
raise ValueError('Info is incomplete: %s' % e)
meta_prefix = 'x_' + db_type + '_'
for key, value in info.iteritems():
@ -152,6 +237,53 @@ def print_db_info_metadata(db_type, info, metadata):
print 'No user metadata found in db file'
def print_obj_metadata(metadata):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:raises: ValueError
"""
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = metadata.pop('X-Timestamp', 0)
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print 'Path: %s' % path
print ' Account: %s' % account
print ' Container: %s' % container
print ' Object: %s' % obj
print ' Object hash: %s' % obj_hash
else:
print 'Path: Not found in metadata'
if content_type:
print 'Content-Type: %s' % content_type
else:
print 'Content-Type: Not found in metadata'
if ts:
print ('Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(ts)), ts))
else:
print 'Timestamp: Not found in metadata'
print 'User Metadata: %s' % metadata
def print_info(db_type, db_file, swift_dir='/etc/swift'):
if db_type not in ('account', 'container'):
print "Unrecognized DB type: internal error"
@ -184,3 +316,201 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
policy_name=''):
"""
Display information about an object read from the datafile.
Optionally verify the datafile content matches the ETag metadata.
:param datafile: path on disk to object file
:param check_etag: boolean, will read datafile content and verify
computed checksum matches value stored in
metadata.
:param swift_dir: the path on disk to rings
:param policy_name: optionally the name to use when finding the ring
"""
if not os.path.exists(datafile) or not datafile.endswith('.data'):
print "Data file doesn't exist"
raise InfoSystemExit()
if not datafile.startswith(('/', './')):
datafile = './' + datafile
policy_index = None
ring = None
datadir = DATADIR_BASE
# try to extract policy index from datafile disk path
try:
policy_index = extract_policy_index(datafile)
except ValueError:
pass
try:
if policy_index:
datadir += '-' + str(policy_index)
ring = Ring(swift_dir, ring_name='object-' + str(policy_index))
elif policy_index == 0:
ring = Ring(swift_dir, ring_name='object')
except IOError:
# no such ring
pass
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
policy_index_for_name = policy.idx
if (policy_index is not None and
policy_index_for_name is not None and
policy_index != policy_index_for_name):
print 'Attention: Ring does not match policy!'
print 'Double check your policy name!'
if not ring and policy_index_for_name:
ring = POLICIES.get_object_ring(policy_index_for_name,
swift_dir)
datadir = get_data_dir(policy_index_for_name)
with open(datafile, 'rb') as fp:
try:
metadata = read_metadata(fp)
except EOFError:
print "Invalid metadata"
raise InfoSystemExit()
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
path = metadata.get('name', '')
print_obj_metadata(metadata)
# Optional integrity check; it's useful, but slow.
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print ("ETag: %s doesn't match file hash of %s!" %
(etag, h))
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print ("Content-Length: %s doesn't match file length of %s"
% (length, file_len))
else:
print 'Content-Length: Not found in metadata'
account, container, obj = path.split('/', 3)[1:]
if ring:
print_ring_locations(ring, datadir, account, container, obj,
policy_index=policy_index)
def print_item_locations(ring, ring_name=None, account=None, container=None,
obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get('policy_name', None)
part = kwargs.get('partition', None)
all_nodes = kwargs.get('all', False)
swift_dir = kwargs.get('swift_dir', '/etc/swift')
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print 'Attention! mismatch between ring and policy detected!'
else:
print 'Attention! Policy %s is not valid' % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print 'Need a ring or policy'
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print 'No policy named %r' % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print 'No account specified'
raise InfoSystemExit()
if container is None and obj is not None:
print 'No container specified'
raise InfoSystemExit()
if account is None and part is None:
print 'No target specified'
raise InfoSystemExit()
loc = '<type>'
if part and ring_name:
if '-' in ring_name and ring_name.startswith('object'):
loc = 'objects-' + ring_name.split('-', 1)[1]
else:
loc = ring_name + 's'
if account and container and obj:
loc = 'objects'
if '-' in ring_name and ring_name.startswith('object'):
policy_index = int(ring_name.rsplit('-', 1)[1])
loc = 'objects-%d' % policy_index
if account and container and not obj:
loc = 'containers'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='container')
else:
if ring_name != 'container':
print 'Attention! mismatch between ring and item detected!'
if account and not container and not obj:
loc = 'accounts'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='account')
else:
if ring_name != 'account':
print 'Attention! mismatch between ring and item detected!'
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n\n' % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
policy_index=policy_index)

View File

@ -19,17 +19,22 @@ from cStringIO import StringIO
from shutil import rmtree
from tempfile import mkdtemp
from test.unit import write_fake_ring
from test.unit import patch_policies, write_fake_ring
from swift.common import ring, utils
from swift.common.swob import Request
from swift.common.storage_policy import StoragePolicy, POLICIES
from swift.cli.info import print_db_info_metadata, print_ring_locations, \
print_info, InfoSystemExit
print_info, print_obj_metadata, print_obj, InfoSystemExit
from swift.account.server import AccountController
from swift.container.server import ContainerController
from swift.obj.diskfile import write_metadata
class TestCliInfo(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'one', False),
StoragePolicy(2, 'two', False)])
class TestCliInfoBase(unittest.TestCase):
def setUp(self):
self.orig_hp = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
utils.HASH_PATH_PREFIX = 'info'
@ -42,14 +47,30 @@ class TestCliInfo(unittest.TestCase):
utils.mkdirs(os.path.join(self.testdir, 'sdb1'))
utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp'))
self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz')
account_devs = [{'ip': '127.0.0.1', 'port': 42},
{'ip': '127.0.0.2', 'port': 43}]
account_devs = [
{'ip': '127.0.0.1', 'port': 42},
{'ip': '127.0.0.2', 'port': 43},
]
write_fake_ring(self.account_ring_path, *account_devs)
self.container_ring_path = os.path.join(self.testdir,
'container.ring.gz')
container_devs = [{'ip': '127.0.0.3', 'port': 42},
{'ip': '127.0.0.4', 'port': 43}]
container_devs = [
{'ip': '127.0.0.3', 'port': 42},
{'ip': '127.0.0.4', 'port': 43},
]
write_fake_ring(self.container_ring_path, *container_devs)
self.object_ring_path = os.path.join(self.testdir, 'object.ring.gz')
object_devs = [
{'ip': '127.0.0.3', 'port': 42},
{'ip': '127.0.0.4', 'port': 43},
]
write_fake_ring(self.object_ring_path, *object_devs)
# another ring for policy 1
self.one_ring_path = os.path.join(self.testdir, 'object-1.ring.gz')
write_fake_ring(self.one_ring_path, *object_devs)
# ... and another for policy 2
self.two_ring_path = os.path.join(self.testdir, 'object-2.ring.gz')
write_fake_ring(self.two_ring_path, *object_devs)
def tearDown(self):
utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp
@ -59,10 +80,13 @@ class TestCliInfo(unittest.TestCase):
try:
func(*args, **kwargs)
except Exception, e:
self.assertEqual(msg, str(e))
self.assertTrue(msg in str(e),
"Expected %r in %r" % (msg, str(e)))
self.assertTrue(isinstance(e, exc),
"Expected %s, got %s" % (exc, type(e)))
class TestCliInfo(TestCliInfoBase):
def test_print_db_info_metadata(self):
self.assertRaisesMessage(ValueError, 'Wrong DB type',
print_db_info_metadata, 't', {}, {})
@ -76,6 +100,7 @@ class TestCliInfo(unittest.TestCase):
created_at=100.1,
put_timestamp=106.3,
delete_timestamp=107.9,
status_changed_at=108.3,
container_count='3',
object_count='20',
bytes_used='42')
@ -93,6 +118,7 @@ Metadata:
Created at: 1970-01-01 00:01:40.100000 (100.1)
Put Timestamp: 1970-01-01 00:01:46.300000 (106.3)
Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9)
Status Timestamp: 1970-01-01 00:01:48.300000 (108.3)
Container Count: 3
Object Count: 20
Bytes Used: 42
@ -108,9 +134,11 @@ No system metadata found in db file
info = dict(
account='acct',
container='cont',
storage_policy_index=0,
created_at='0000000100.10000',
put_timestamp='0000000106.30000',
delete_timestamp='0000000107.90000',
status_changed_at='0000000108.30000',
object_count='20',
bytes_used='42',
reported_put_timestamp='0000010106.30000',
@ -133,8 +161,10 @@ Metadata:
Created at: 1970-01-01 00:01:40.100000 (0000000100.10000)
Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000)
Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000)
Status Timestamp: 1970-01-01 00:01:48.300000 (0000000108.30000)
Object Count: 20
Bytes Used: 42
Storage Policy: %s (0)
Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000)
Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000)
Reported Object Count: 20
@ -144,54 +174,62 @@ Metadata:
X-Container-Bar: goo
X-Container-Foo: bar
System Metadata: {'mydata': 'swift'}
No user metadata found in db file'''
No user metadata found in db file''' % POLICIES[0].name
self.assertEquals(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.split('\n')))
def test_print_ring_locations(self):
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
def test_print_ring_locations_invalid_args(self):
self.assertRaises(ValueError, print_ring_locations,
None, 'dir', 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
self.assertRaises(ValueError, print_ring_locations,
[], None, 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', None)
self.assertRaisesMessage(ValueError, 'Ring error',
print_ring_locations,
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', 'acct', 'con')
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', 'acct', obj='o')
def test_print_ring_locations_account(self):
out = StringIO()
with mock.patch('sys.stdout', out):
acctring = ring.Ring(self.testdir, ring_name='account')
print_ring_locations(acctring, 'dir', 'acct')
exp_db2 = os.path.join('/srv', 'node', 'sdb1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_db1 = os.path.join('/srv', 'node', 'sda1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_out = ('Ring locations:\n 127.0.0.2:43 - %s\n'
' 127.0.0.1:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the account config file on'
' each storage node.' % (exp_db2, exp_db1))
self.assertEquals(out.getvalue().strip(), exp_out)
exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '3',
'b47', 'dc5be2aa4347a22a0fee6bc7de505b47')
self.assertTrue(exp_db in out.getvalue())
self.assertTrue('127.0.0.1' in out.getvalue())
self.assertTrue('127.0.0.2' in out.getvalue())
def test_print_ring_locations_container(self):
out = StringIO()
with mock.patch('sys.stdout', out):
contring = ring.Ring(self.testdir, ring_name='container')
print_ring_locations(contring, 'dir', 'acct', 'con')
exp_db4 = os.path.join('/srv', 'node', 'sdb1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_db3 = os.path.join('/srv', 'node', 'sda1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_out = ('Ring locations:\n 127.0.0.4:43 - %s\n'
' 127.0.0.3:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the container config file on'
' each storage node.' % (exp_db4, exp_db3))
self.assertEquals(out.getvalue().strip(), exp_out)
exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '1',
'fe6', '63e70955d78dfc62821edc07d6ec1fe6')
self.assertTrue(exp_db in out.getvalue())
def test_print_ring_locations_obj(self):
out = StringIO()
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
print_ring_locations(objring, 'dir', 'acct', 'con', 'obj')
exp_obj = os.path.join('${DEVICE:-/srv/node*}', 'sda1', 'dir', '1',
'117', '4a16154fc15c75e26ba6afadf5b1c117')
self.assertTrue(exp_obj in out.getvalue())
def test_print_ring_locations_partition_number(self):
out = StringIO()
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
print_ring_locations(objring, 'objects', None, tpart='1')
exp_obj1 = os.path.join('${DEVICE:-/srv/node*}', 'sda1',
'objects', '1')
exp_obj2 = os.path.join('${DEVICE:-/srv/node*}', 'sdb1',
'objects', '1')
self.assertTrue(exp_obj1 in out.getvalue())
self.assertTrue(exp_obj2 in out.getvalue())
def test_print_info(self):
db_file = 'foo'
@ -271,3 +309,199 @@ No user metadata found in db file'''
self.assertEquals(out.getvalue().strip(), exp_out)
else:
self.fail("Expected an InfoSystemExit exception to be raised")
class TestPrintObj(TestCliInfoBase):
def setUp(self):
super(TestPrintObj, self).setUp()
self.datafile = os.path.join(self.testdir,
'1402017432.46642.data')
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream'}
write_metadata(fp, md)
def test_print_obj_invalid(self):
datafile = '1402017324.68634.data'
self.assertRaises(InfoSystemExit, print_obj, datafile)
datafile = os.path.join(self.testdir, './1234.data')
self.assertRaises(InfoSystemExit, print_obj, datafile)
with open(datafile, 'wb') as fp:
fp.write('1234')
out = StringIO()
with mock.patch('sys.stdout', out):
self.assertRaises(InfoSystemExit, print_obj, datafile)
self.assertEquals(out.getvalue().strip(),
'Invalid metadata')
def test_print_obj_valid(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir)
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
def test_print_obj_with_policy(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir, policy_name='one')
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
ring_loc_msg = 'ls -lah'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
self.assertTrue(ring_loc_msg in out.getvalue())
def test_missing_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: Not found in metadata' in out.getvalue())
class TestPrintObjFullMeta(TestCliInfoBase):
def setUp(self):
super(TestPrintObjFullMeta, self).setUp()
self.datafile = os.path.join(self.testdir,
'sda', 'objects-1',
'1', 'ea8',
'db4449e025aca992307c7c804a67eea8',
'1402017884.18202.data')
utils.mkdirs(os.path.dirname(self.datafile))
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
write_metadata(fp, md)
def test_print_obj(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir)
self.assertTrue('/objects-1/' in out.getvalue())
def test_print_obj_no_ring(self):
no_rings_dir = os.path.join(self.testdir, 'no_rings_here')
os.mkdir(no_rings_dir)
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=no_rings_dir)
self.assertTrue('d41d8cd98f00b204e9800998ecf8427e' in out.getvalue())
self.assertTrue('Partition' not in out.getvalue())
def test_print_obj_policy_name_mismatch(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, policy_name='two', swift_dir=self.testdir)
ring_alert_msg = 'Attention: Ring does not match policy'
self.assertTrue(ring_alert_msg in out.getvalue())
def test_valid_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (valid)'
in out.getvalue())
def test_invalid_etag(self):
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'badetag',
'Content-Length': 0}
write_metadata(fp, md)
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: badetag doesn\'t match file hash'
in out.getvalue())
def test_unchecked_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, check_etag=False)
self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (not checked)'
in out.getvalue())
def test_print_obj_metadata(self):
self.assertRaisesMessage(ValueError, 'Metadata is None',
print_obj_metadata, [])
def reset_metadata():
md = dict(name='/AUTH_admin/c/dummy')
md['Content-Type'] = 'application/octet-stream'
md['X-Timestamp'] = 106.3
md['X-Object-Meta-Mtime'] = '107.3'
return md
metadata = reset_metadata()
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: application/octet-stream
Timestamp: 1970-01-01 00:01:46.300000 (106.3)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}'''
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
metadata['name'] = '/a-s'
self.assertRaisesMessage(ValueError, 'Path is invalid',
print_obj_metadata, metadata)
metadata = reset_metadata()
del metadata['name']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: Not found in metadata
Content-Type: application/octet-stream
Timestamp: 1970-01-01 00:01:46.300000 (106.3)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}'''
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
del metadata['Content-Type']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: Not found in metadata
Timestamp: 1970-01-01 00:01:46.300000 (106.3)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}'''
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
del metadata['X-Timestamp']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: application/octet-stream
Timestamp: Not found in metadata
User Metadata: {'X-Object-Meta-Mtime': '107.3'}'''
self.assertEquals(out.getvalue().strip(), exp_out)