From cb584303218805d1c1374d732caac26457a6ffe8 Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 2 Feb 2011 13:39:08 -0800 Subject: [PATCH] logging: use routes to separate logging configurations --- bin/swift-drive-audit | 2 +- bin/swift-log-uploader | 2 +- swift/account/auditor.py | 2 +- swift/account/reaper.py | 2 +- swift/account/server.py | 2 +- swift/auth/server.py | 2 +- swift/common/daemon.py | 4 ++-- swift/common/db_replicator.py | 2 +- swift/common/middleware/catch_errors.py | 2 +- swift/common/middleware/cname_lookup.py | 2 +- swift/common/middleware/ratelimit.py | 2 +- swift/common/middleware/swauth.py | 2 +- swift/common/utils.py | 2 ++ swift/common/wsgi.py | 2 +- swift/container/auditor.py | 2 +- swift/container/server.py | 2 +- swift/container/updater.py | 2 +- swift/obj/auditor.py | 2 +- swift/obj/replicator.py | 2 +- swift/obj/server.py | 2 +- swift/obj/updater.py | 2 +- swift/proxy/server.py | 2 +- swift/stats/access_processor.py | 2 +- swift/stats/account_stats.py | 3 ++- swift/stats/log_processor.py | 4 ++-- swift/stats/log_uploader.py | 3 ++- swift/stats/stats_processor.py | 2 +- test/unit/auth/test_server.py | 4 ++-- test/unit/common/test_daemon.py | 4 ++-- test/unit/common/test_utils.py | 8 +++++--- 30 files changed, 41 insertions(+), 35 deletions(-) diff --git a/bin/swift-drive-audit b/bin/swift-drive-audit index e92d1e3c12..5203f54b6b 100755 --- a/bin/swift-drive-audit +++ b/bin/swift-drive-audit @@ -99,7 +99,7 @@ if __name__ == '__main__': device_dir = conf.get('device_dir', '/srv/node') minutes = int(conf.get('minutes', 60)) error_limit = int(conf.get('error_limit', 1)) - logger = get_logger(conf, 'drive-audit') + logger = get_logger(conf, log_route='drive-audit') devices = get_devices(device_dir, logger) logger.debug("Devices found: %s" % str(devices)) if not devices: diff --git a/bin/swift-log-uploader b/bin/swift-log-uploader index 9d0e27836c..93cb8f6f97 100755 --- a/bin/swift-log-uploader +++ b/bin/swift-log-uploader @@ -34,7 +34,7 @@ if __name__ == '__main__': uploader_conf.update(plugin_conf) # pre-configure logger - logger = utils.get_logger(uploader_conf, plugin, + logger = utils.get_logger(uploader_conf, plugin, log_route='log-uploader', log_to_console=options.get('verbose', False)) # currently LogUploader only supports run_once options['once'] = True diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 1f24f93acc..63551354d8 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -28,7 +28,7 @@ class AccountAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'account-auditor') + self.logger = get_logger(conf, log_route='account-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/account/reaper.py b/swift/account/reaper.py index dd0d4b3890..ba78db8d98 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -53,7 +53,7 @@ class AccountReaper(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='account-reaper') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/account/server.py b/swift/account/server.py index 2c83f51cc6..f15ac38c11 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -42,7 +42,7 @@ class AccountController(object): """WSGI controller for the account server.""" def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='account-server') self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/auth/server.py b/swift/auth/server.py index f9cd56dd0e..1258a706e7 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -90,7 +90,7 @@ class AuthController(object): """ def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='auth-server') self.super_admin_key = conf.get('super_admin_key') if not self.super_admin_key: msg = _('No super_admin_key set in conf file! Exiting.') diff --git a/swift/common/daemon.py b/swift/common/daemon.py index eee3428679..91230e4d2b 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -26,7 +26,7 @@ class Daemon(object): def __init__(self, conf): self.conf = conf - self.logger = utils.get_logger(conf, 'swift-daemon') + self.logger = utils.get_logger(conf, log_route='daemon') def run_once(self): """Override this to run the script once""" @@ -84,7 +84,7 @@ def run_daemon(klass, conf_file, section_name='', logger = kwargs.pop('logger') else: logger = utils.get_logger(conf, conf.get('log_name', section_name), - log_to_console=kwargs.pop('verbose', False)) + log_to_console=kwargs.pop('verbose', False), log_route=section_name) try: klass(conf).run(once=once, **kwargs) except KeyboardInterrupt: diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 49756f1f7b..3c3731d45a 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -92,7 +92,7 @@ class Replicator(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='replicator') self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 10d8614194..16ade84689 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -30,7 +30,7 @@ class CatchErrorMiddleware(object): self.logger = getattr(app, 'logger', None) if not self.logger: # and only call get_logger if we have to - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='catch-errors') def __call__(self, env, start_response): try: diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py index f13155c1fe..8ea9f88071 100644 --- a/swift/common/middleware/cname_lookup.py +++ b/swift/common/middleware/cname_lookup.py @@ -53,7 +53,7 @@ class CNAMELookupMiddleware(object): self.storage_domain = '.' + self.storage_domain self.lookup_depth = int(conf.get('lookup_depth', '1')) self.memcache = None - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='cname-lookup') def __call__(self, env, start_response): if not self.storage_domain: diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 4657b6abcd..485b1db26e 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -39,7 +39,7 @@ class RateLimitMiddleware(object): if logger: self.logger = logger else: - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='ratelimit') self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 5965e710ac..32328e8eb6 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -51,7 +51,7 @@ class Swauth(object): def __init__(self, app, conf): self.app = app self.conf = conf - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='swauth') self.log_headers = conf.get('log_headers') == 'True' self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': diff --git a/swift/common/utils.py b/swift/common/utils.py index 8da3d1f8f4..698fae2cc1 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -395,6 +395,8 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None): :param conf: Configuration dict to read settings from :param name: Name of the logger :param log_to_console: Add handler which writes to console on stderr + :param log_route: Route for the logging, not emitted to the log, just used + to separate logging configurations """ if not conf: conf = {} diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 9450bcf439..46207fe8e1 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -113,7 +113,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs): logger = kwargs.pop('logger') else: logger = get_logger(conf, log_name, - log_to_console=kwargs.pop('verbose', False)) + log_to_console=kwargs.pop('verbose', False), log_route='wsgi') # redirect errors to logger and close stdio capture_stdio(logger) diff --git a/swift/container/auditor.py b/swift/container/auditor.py index d1ceb4f98a..0b1c10e03e 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -28,7 +28,7 @@ class ContainerAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'container-auditor') + self.logger = get_logger(conf, log_route='container-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/container/server.py b/swift/container/server.py index cfcdded1e4..561dad2ea9 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -49,7 +49,7 @@ class ContainerController(object): save_headers = ['x-container-read', 'x-container-write'] def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='container-server') self.root = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/container/updater.py b/swift/container/updater.py index 883dd17101..0bd000f3f2 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -37,7 +37,7 @@ class ContainerUpdater(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'container-updater') + self.logger = get_logger(conf, log_route='container-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 09fdd77774..8ed05049f3 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -31,7 +31,7 @@ class ObjectAuditor(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'object-auditor') + self.logger = get_logger(conf, log_route='object-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index dcfcb926f9..8dec8aa801 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -207,7 +207,7 @@ class ObjectReplicator(Daemon): :param logger: logging object """ self.conf = conf - self.logger = get_logger(conf, 'object-replicator') + self.logger = get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/server.py b/swift/obj/server.py index f2e2b31314..e3626bf692 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -266,7 +266,7 @@ class ObjectController(object): /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='object-server') self.devices = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 2b28ff08c5..356be64da4 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -35,7 +35,7 @@ class ObjectUpdater(Daemon): def __init__(self, conf): self.conf = conf - self.logger = get_logger(conf, 'object-updater') + self.logger = get_logger(conf, log_route='object-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1eae0dfc30..a66f643a68 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1607,7 +1607,7 @@ class BaseApplication(object): def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if logger is None: - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger if conf is None: diff --git a/swift/stats/access_processor.py b/swift/stats/access_processor.py index 2aee505415..6965ef2b4a 100644 --- a/swift/stats/access_processor.py +++ b/swift/stats/access_processor.py @@ -34,7 +34,7 @@ class AccessLogProcessor(object): conf.get('service_ips', '').split(',')\ if x.strip()] self.warn_percent = float(conf.get('warn_percent', '0.8')) - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='access-processor') def log_line_parser(self, raw_log): '''given a raw access log line, return a dict of the good parts''' diff --git a/swift/stats/account_stats.py b/swift/stats/account_stats.py index 325746386c..34b024d2c2 100644 --- a/swift/stats/account_stats.py +++ b/swift/stats/account_stats.py @@ -48,7 +48,8 @@ class AccountStat(Daemon): self.devices = server_conf.get('devices', '/srv/node') self.mount_check = server_conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.logger = get_logger(stats_conf, 'swift-account-stats-logger') + self.logger = \ + get_logger(stats_conf, log_route='account-stats') def run_once(self): self.logger.info(_("Gathering account stats")) diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py index 5dbc92afbe..727e687f38 100644 --- a/swift/stats/log_processor.py +++ b/swift/stats/log_processor.py @@ -40,7 +40,7 @@ class LogProcessor(object): def __init__(self, conf, logger): if isinstance(logger, tuple): - self.logger = get_logger(*logger) + self.logger = get_logger(*logger, log_route='log-processor') else: self.logger = logger @@ -226,7 +226,7 @@ class LogProcessorDaemon(Daemon): c = conf.get('log-processor') super(LogProcessorDaemon, self).__init__(c) self.total_conf = conf - self.logger = get_logger(c) + self.logger = get_logger(c, log_route='log-processor') self.log_processor = LogProcessor(conf, self.logger) self.lookback_hours = int(c.get('lookback_hours', '120')) self.lookback_window = int(c.get('lookback_window', diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index b425738938..a828188eb7 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -65,7 +65,8 @@ class LogUploader(Daemon): self.filename_format = source_filename_format self.internal_proxy = InternalProxy(proxy_server_conf) log_name = 'swift-log-uploader-%s' % plugin_name - self.logger = utils.get_logger(uploader_conf, plugin_name) + self.logger = \ + utils.get_logger(uploader_conf, plugin_name, log_route=plugin_name) def run_once(self): self.logger.info(_("Uploading logs")) diff --git a/swift/stats/stats_processor.py b/swift/stats/stats_processor.py index 95dba7604c..f9496c1df9 100644 --- a/swift/stats/stats_processor.py +++ b/swift/stats/stats_processor.py @@ -20,7 +20,7 @@ class StatsLogProcessor(object): """Transform account storage stat logs""" def __init__(self, conf): - self.logger = get_logger(conf) + self.logger = get_logger(conf, log_route='stats-processor') def process(self, obj_stream, data_object_account, data_object_container, data_object_name): diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 4060766d65..d58556ab22 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -456,7 +456,7 @@ class TestAuthServer(unittest.TestCase): def test_basic_logging(self): log = StringIO() log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth') + logger = get_logger(self.conf, 'auth-server', log_route='auth-server') logger.logger.addHandler(log_handler) try: auth_server.http_connect = fake_http_connect(201) @@ -534,7 +534,7 @@ class TestAuthServer(unittest.TestCase): orig_Request = auth_server.Request log = StringIO() log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth') + logger = get_logger(self.conf, 'auth-server', log_route='auth-server') logger.logger.addHandler(log_handler) try: auth_server.Request = request_causing_exception diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index a4addcee51..1d54e78c3e 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -28,7 +28,7 @@ class MyDaemon(daemon.Daemon): def __init__(self, conf): self.conf = conf - self.logger = utils.get_logger(None, 'server') + self.logger = utils.get_logger(None, 'server', log_route='server') MyDaemon.forever_called = False MyDaemon.once_called = False @@ -99,7 +99,7 @@ user = %s sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) - logger = utils.get_logger(None, 'server') + logger = utils.get_logger(None, 'server', log_route='server') daemon.run_daemon(MyDaemon, conf_file, logger=logger) self.assert_('user quit' in sio.getvalue().lower()) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 0c81b15698..8da913c489 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -289,17 +289,19 @@ Error: unable to locate %s sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) - logger = utils.get_logger(None, 'server') + logger = utils.get_logger(None, 'server', log_route='server') logger.warn('test1') self.assertEquals(sio.getvalue(), 'test1\n') logger.debug('test2') self.assertEquals(sio.getvalue(), 'test1\n') - logger = utils.get_logger({'log_level': 'DEBUG'}, 'server') + logger = utils.get_logger({'log_level': 'DEBUG'}, 'server', + log_route='server') logger.debug('test3') self.assertEquals(sio.getvalue(), 'test1\ntest3\n') # Doesn't really test that the log facility is truly being used all the # way to syslog; but exercises the code. - logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server') + logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server', + log_route='server') logger.warn('test4') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n')