From 9e80fd45a0072e15e9d49299a6fa9e9c725e2207 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 18 Nov 2013 11:41:58 -0800 Subject: [PATCH] Add a DebugLogger for wsgi server tests Change-Id: Ifd2528be443ba3879bf4921f6c5f4ef31f29044b --- swift/account/server.py | 4 ++-- swift/common/middleware/proxy_logging.py | 6 ++--- swift/container/server.py | 4 ++-- swift/obj/server.py | 4 ++-- test/unit/__init__.py | 30 ++++++++++++++++++++---- test/unit/obj/test_server.py | 5 ++-- test/unit/proxy/test_server.py | 29 ++++++++++++++++------- 7 files changed, 58 insertions(+), 24 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 7a41564fca..e975f9db10 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -47,8 +47,8 @@ DATADIR = 'accounts' class AccountController(object): """WSGI controller for the account server.""" - def __init__(self, conf): - self.logger = get_logger(conf, log_route='account-server') + def __init__(self, conf, logger=None): + self.logger = logger or get_logger(conf, log_route='account-server') self.root = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) replication_server = conf.get('replication_server', None) diff --git a/swift/common/middleware/proxy_logging.py b/swift/common/middleware/proxy_logging.py index 8a65bc4b1b..2a4de5f8c7 100644 --- a/swift/common/middleware/proxy_logging.py +++ b/swift/common/middleware/proxy_logging.py @@ -87,7 +87,7 @@ class ProxyLoggingMiddleware(object): Middleware that logs Swift proxy requests in the swift log format. """ - def __init__(self, app, conf): + def __init__(self, app, conf, logger=None): self.app = app self.log_hdrs = config_true_value(conf.get( 'access_log_headers', @@ -110,8 +110,8 @@ class ProxyLoggingMiddleware(object): value = conf.get('access_' + key, conf.get(key, None)) if value: access_log_conf[key] = value - self.access_logger = get_logger(access_log_conf, - log_route='proxy-access') + self.access_logger = logger or get_logger(access_log_conf, + log_route='proxy-access') self.access_logger.set_statsd_prefix('proxy-server') self.reveal_sensitive_prefix = int(conf.get('reveal_sensitive_prefix', MAX_HEADER_SIZE)) diff --git a/swift/container/server.py b/swift/container/server.py index 4810b9bfaf..300e3f6ba2 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -54,8 +54,8 @@ class ContainerController(object): save_headers = ['x-container-read', 'x-container-write', 'x-container-sync-key', 'x-container-sync-to'] - def __init__(self, conf): - self.logger = get_logger(conf, log_route='container-server') + def __init__(self, conf, logger=None): + self.logger = logger or get_logger(conf, log_route='container-server') self.root = conf.get('devices', '/srv/node/') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.node_timeout = int(conf.get('node_timeout', 3)) diff --git a/swift/obj/server.py b/swift/obj/server.py index 293909704c..0163f77126 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -51,14 +51,14 @@ from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" - def __init__(self, conf): + def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ - self.logger = get_logger(conf, log_route='object-server') + self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 0d07493b04..2040ba63dc 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -19,7 +19,7 @@ import os import copy import logging import errno -from sys import exc_info +import sys from contextlib import contextmanager from collections import defaultdict from tempfile import NamedTemporaryFile @@ -28,7 +28,7 @@ from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree from test import get_config -from swift.common.utils import config_true_value +from swift.common.utils import config_true_value, LogAdapter from hashlib import md5 from eventlet import sleep, Timeout import logging.handlers @@ -234,6 +234,7 @@ class FakeLogger(logging.Logger): self.level = logging.NOTSET if 'facility' in kwargs: self.facility = kwargs['facility'] + self.statsd_client = None def _clear(self): self.log_dict = defaultdict(list) @@ -259,7 +260,8 @@ class FakeLogger(logging.Logger): debug = _store_and_log_in('debug') def exception(self, *args, **kwargs): - self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) + self.log_dict['exception'].append((args, kwargs, + str(sys.exc_info()[1]))) print 'FakeLogger Exception: %s' % self.log_dict # mock out the StatsD logging methods: @@ -303,7 +305,7 @@ class FakeLogger(logging.Logger): def emit(self, record): pass - def handle(self, record): + def _handle(self, record): try: line = record.getMessage() except TypeError: @@ -312,6 +314,9 @@ class FakeLogger(logging.Logger): raise self.lines_dict[record.levelno].append(line) + def handle(self, record): + self._handle(record) + def flush(self): pass @@ -319,6 +324,23 @@ class FakeLogger(logging.Logger): pass +class DebugLogger(FakeLogger): + """A simple stdout logging version of FakeLogger""" + + def __init__(self, *args, **kwargs): + FakeLogger.__init__(self, *args, **kwargs) + self.formatter = logging.Formatter("%(server)s: %(message)s") + + def handle(self, record): + self._handle(record) + print self.formatter.format(record) + + +def debug_logger(name='test'): + """get a named adapted debug logger""" + return LogAdapter(DebugLogger(), name) + + original_syslog_handler = logging.handlers.SysLogHandler diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 56c97917a6..0833146376 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -28,7 +28,7 @@ from tempfile import mkdtemp from hashlib import md5 from eventlet import sleep, spawn, wsgi, listen, Timeout -from test.unit import FakeLogger +from test.unit import FakeLogger, debug_logger from test.unit import connect_tcp, readuntil2crlfs from swift.obj import server as object_server from swift.obj import diskfile @@ -55,7 +55,8 @@ class TestObjectController(unittest.TestCase): os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) conf = {'devices': self.testdir, 'mount_check': 'false'} - self.object_controller = object_server.ObjectController(conf) + self.object_controller = object_server.ObjectController( + conf, logger=debug_logger()) self.object_controller.bytes_per_sync = 1 self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 72a813cf92..ddf1568e5e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -35,12 +35,13 @@ from eventlet import sleep, spawn, wsgi, listen import simplejson from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ - fake_http_connect, FakeRing, FakeMemcache + fake_http_connect, FakeRing, FakeMemcache, debug_logger from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server from swift.common import ring +from swift.common.middleware import proxy_logging from swift.common.exceptions import ChunkReadTimeout, SegmentError from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ @@ -130,17 +131,26 @@ def do_setup(the_object_server): {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}], 30), f) - prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone()) - acc1srv = account_server.AccountController(conf) - acc2srv = account_server.AccountController(conf) - con1srv = container_server.ContainerController(conf) - con2srv = container_server.ContainerController(conf) - obj1srv = the_object_server.ObjectController(conf) - obj2srv = the_object_server.ObjectController(conf) + prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(), + logger=debug_logger('proxy')) + acc1srv = account_server.AccountController( + conf, logger=debug_logger('acct1')) + acc2srv = account_server.AccountController( + conf, logger=debug_logger('acct2')) + con1srv = container_server.ContainerController( + conf, logger=debug_logger('cont1')) + con2srv = container_server.ContainerController( + conf, logger=debug_logger('cont2')) + obj1srv = the_object_server.ObjectController( + conf, logger=debug_logger('obj1')) + obj2srv = the_object_server.ObjectController( + conf, logger=debug_logger('obj2')) _test_servers = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) nl = NullLogger() - prospa = spawn(wsgi.server, prolis, prosrv, nl) + logging_prosv = proxy_logging.ProxyLoggingMiddleware(prosrv, conf, + logger=prosrv.logger) + prospa = spawn(wsgi.server, prolis, logging_prosv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) con1spa = spawn(wsgi.server, con1lis, con1srv, nl) @@ -4584,6 +4594,7 @@ class TestObjectController(unittest.TestCase): fd.read(1) fd.close() sock.close() + sleep(0) # let eventlet do it's thing # Make sure the GC is run again for pythons without reference counting for i in xrange(4): gc.collect()