3d3ed34f44
Documentation, including a list of metrics reported and their semantics, is in the Admin Guide in a new section, "Reporting Metrics to StatsD". An optional "metric prefix" may be configured which will be prepended to every metric name sent to StatsD. Here is the rationale for doing a deep integration like this versus only sending metrics to StatsD in middleware. It's the only way to report some internal activities of Swift in a real-time manner. So to have one way of reporting to StatsD and one place/style of configuration, even some things (like, say, timing of PUT requests into the proxy-server) which could be logged via middleware are consistently logged the same way (deep integration via the logger delegate methods). When log_statsd_host is configured, get_logger() injects a swift.common.utils.StatsdClient object into the logger as logger.statsd_client. Then a set of delegate methods on LogAdapter either pass through to the StatsdClient object or become no-ops. This allows StatsD logging to look like: self.logger.increment('some.metric.here') and do the right thing in all cases and with no messy conditional logic. I wanted to use the pystatsd module for the StatsD client, but the version on PyPi is lagging the git repo (and is missing both the prefix functionality and timing_since() method). So I wrote my swift.common.utils.StatsdClient. The interface is the same as pystatsd.Client, but the code was written from scratch. It's pretty simple, and the tests I added cover it. This also frees Swift from an optional dependency on the pystatsd module, making this feature easier to enable. There's test coverage for the new code and all existing tests continue to pass. Refactored out _one_audit_pass() method in swift/account/auditor.py and swift/container/auditor.py. Fixed some misc. PEP8 violations. Misc test cleanups and refactorings (particularly the way "fake logging" is handled). Change-Id: Ie968a9ae8771f59ee7591e2ae11999c44bfe33b2
238 lines
5.2 KiB
Python
238 lines
5.2 KiB
Python
""" Swift tests """
|
|
|
|
import sys
|
|
import os
|
|
import copy
|
|
import logging
|
|
from sys import exc_info
|
|
from contextlib import contextmanager
|
|
from tempfile import NamedTemporaryFile
|
|
from eventlet.green import socket
|
|
from tempfile import mkdtemp
|
|
from shutil import rmtree
|
|
from test import get_config
|
|
from ConfigParser import MissingSectionHeaderError
|
|
from StringIO import StringIO
|
|
from swift.common.utils import readconf, TRUE_VALUES
|
|
from logging import Handler
|
|
import logging.handlers
|
|
|
|
|
|
def readuntil2crlfs(fd):
|
|
rv = ''
|
|
lc = ''
|
|
crlfs = 0
|
|
while crlfs < 2:
|
|
c = fd.read(1)
|
|
rv = rv + c
|
|
if c == '\r' and lc != '\n':
|
|
crlfs = 0
|
|
if lc == '\r' and c == '\n':
|
|
crlfs += 1
|
|
lc = c
|
|
return rv
|
|
|
|
|
|
def connect_tcp(hostport):
|
|
rv = socket.socket()
|
|
rv.connect(hostport)
|
|
return rv
|
|
|
|
|
|
@contextmanager
|
|
def tmpfile(content):
|
|
with NamedTemporaryFile('w', delete=False) as f:
|
|
file_name = f.name
|
|
f.write(str(content))
|
|
try:
|
|
yield file_name
|
|
finally:
|
|
os.unlink(file_name)
|
|
|
|
xattr_data = {}
|
|
|
|
|
|
def _get_inode(fd):
|
|
if not isinstance(fd, int):
|
|
try:
|
|
fd = fd.fileno()
|
|
except AttributeError:
|
|
return os.stat(fd).st_ino
|
|
return os.fstat(fd).st_ino
|
|
|
|
|
|
def _setxattr(fd, k, v):
|
|
inode = _get_inode(fd)
|
|
data = xattr_data.get(inode, {})
|
|
data[k] = v
|
|
xattr_data[inode] = data
|
|
|
|
|
|
def _getxattr(fd, k):
|
|
inode = _get_inode(fd)
|
|
data = xattr_data.get(inode, {}).get(k)
|
|
if not data:
|
|
raise IOError
|
|
return data
|
|
|
|
import xattr
|
|
xattr.setxattr = _setxattr
|
|
xattr.getxattr = _getxattr
|
|
|
|
|
|
@contextmanager
|
|
def temptree(files, contents=''):
|
|
# generate enough contents to fill the files
|
|
c = len(files)
|
|
contents = (list(contents) + [''] * c)[:c]
|
|
tempdir = mkdtemp()
|
|
for path, content in zip(files, contents):
|
|
if os.path.isabs(path):
|
|
path = '.' + path
|
|
new_path = os.path.join(tempdir, path)
|
|
subdir = os.path.dirname(new_path)
|
|
if not os.path.exists(subdir):
|
|
os.makedirs(subdir)
|
|
with open(new_path, 'w') as f:
|
|
f.write(str(content))
|
|
try:
|
|
yield tempdir
|
|
finally:
|
|
rmtree(tempdir)
|
|
|
|
|
|
class NullLoggingHandler(logging.Handler):
|
|
|
|
def emit(self, record):
|
|
pass
|
|
|
|
|
|
class FakeLogger(object):
|
|
# a thread safe logger
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self._clear()
|
|
self.level = logging.NOTSET
|
|
if 'facility' in kwargs:
|
|
self.facility = kwargs['facility']
|
|
|
|
def _clear(self):
|
|
self.log_dict = dict(
|
|
error=[], info=[], warning=[], debug=[], exception=[])
|
|
|
|
def error(self, *args, **kwargs):
|
|
self.log_dict['error'].append((args, kwargs))
|
|
|
|
def info(self, *args, **kwargs):
|
|
self.log_dict['info'].append((args, kwargs))
|
|
|
|
def warning(self, *args, **kwargs):
|
|
self.log_dict['warning'].append((args, kwargs))
|
|
|
|
def debug(self, *args, **kwargs):
|
|
self.log_dict['debug'].append((args, kwargs))
|
|
|
|
def exception(self, *args, **kwargs):
|
|
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
|
|
|
|
# mock out the StatsD logging methods:
|
|
def set_statsd_prefix(self, *a, **kw):
|
|
pass
|
|
|
|
increment = decrement = timing = timing_since = update_stats = \
|
|
set_statsd_prefix
|
|
|
|
def setFormatter(self, obj):
|
|
self.formatter = obj
|
|
|
|
def close(self):
|
|
self._clear()
|
|
|
|
def set_name(self, name):
|
|
# don't touch _handlers
|
|
self._name = name
|
|
|
|
def acquire(self):
|
|
pass
|
|
|
|
def release(self):
|
|
pass
|
|
|
|
def createLock(self):
|
|
pass
|
|
|
|
def emit(self, record):
|
|
pass
|
|
|
|
def handle(self, record):
|
|
pass
|
|
|
|
def flush(self):
|
|
pass
|
|
|
|
def handleError(self, record):
|
|
pass
|
|
|
|
|
|
original_syslog_handler = logging.handlers.SysLogHandler
|
|
|
|
|
|
def fake_syslog_handler():
|
|
for attr in dir(original_syslog_handler):
|
|
if attr.startswith('LOG'):
|
|
setattr(FakeLogger, attr,
|
|
copy.copy(getattr(logging.handlers.SysLogHandler, attr)))
|
|
FakeLogger.priority_map = \
|
|
copy.deepcopy(logging.handlers.SysLogHandler.priority_map)
|
|
|
|
logging.handlers.SysLogHandler = FakeLogger
|
|
|
|
|
|
if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES:
|
|
fake_syslog_handler()
|
|
|
|
|
|
class MockTrue(object):
|
|
"""
|
|
Instances of MockTrue evaluate like True
|
|
Any attr accessed on an instance of MockTrue will return a MockTrue
|
|
instance. Any method called on an instance of MockTrue will return
|
|
a MockTrue instance.
|
|
|
|
>>> thing = MockTrue()
|
|
>>> thing
|
|
True
|
|
>>> thing == True # True == True
|
|
True
|
|
>>> thing == False # True == False
|
|
False
|
|
>>> thing != True # True != True
|
|
False
|
|
>>> thing != False # True != False
|
|
True
|
|
>>> thing.attribute
|
|
True
|
|
>>> thing.method()
|
|
True
|
|
>>> thing.attribute.method()
|
|
True
|
|
>>> thing.method().attribute
|
|
True
|
|
|
|
"""
|
|
|
|
def __getattribute__(self, *args, **kwargs):
|
|
return self
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
return self
|
|
|
|
def __repr__(*args, **kwargs):
|
|
return repr(True)
|
|
|
|
def __eq__(self, other):
|
|
return other is True
|
|
|
|
def __ne__(self, other):
|
|
return other is not True
|