logging: use routes to separate logging configurations

bin/swift-drive-audit

@@ -99,7 +99,7 @@ if __name__ == '__main__':
     device_dir = conf.get('device_dir', '/srv/node')
     minutes = int(conf.get('minutes', 60))
     error_limit = int(conf.get('error_limit', 1))
-    logger = get_logger(conf, 'drive-audit')
+    logger = get_logger(conf, log_route='drive-audit')
     devices = get_devices(device_dir, logger)
     logger.debug("Devices found: %s" % str(devices))
     if not devices:

bin/swift-log-uploader

@@ -34,7 +34,7 @@ if __name__ == '__main__':
     uploader_conf.update(plugin_conf)
 
     # pre-configure logger
-    logger = utils.get_logger(uploader_conf, plugin,
+    logger = utils.get_logger(uploader_conf, plugin, log_route='log-uploader',
                               log_to_console=options.get('verbose', False))
     # currently LogUploader only supports run_once
     options['once'] = True

swift/account/auditor.py

@@ -28,7 +28,7 @@ class AccountAuditor(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf, 'account-auditor')
+        self.logger = get_logger(conf, log_route='account-auditor')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/account/reaper.py

@@ -53,7 +53,7 @@ class AccountReaper(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='account-reaper')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/account/server.py

@@ -42,7 +42,7 @@ class AccountController(object):
     """WSGI controller for the account server."""
 
     def __init__(self, conf):
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='account-server')
         self.root = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/auth/server.py

@@ -90,7 +90,7 @@ class AuthController(object):
     """
 
     def __init__(self, conf):
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='auth-server')
         self.super_admin_key = conf.get('super_admin_key')
         if not self.super_admin_key:
             msg = _('No super_admin_key set in conf file! Exiting.')

swift/common/daemon.py

@@ -26,7 +26,7 @@ class Daemon(object):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = utils.get_logger(conf, 'swift-daemon')
+        self.logger = utils.get_logger(conf, log_route='daemon')
 
     def run_once(self):
         """Override this to run the script once"""
@@ -84,7 +84,7 @@ def run_daemon(klass, conf_file, section_name='',
         logger = kwargs.pop('logger')
     else:
         logger = utils.get_logger(conf, conf.get('log_name', section_name),
-                                  log_to_console=kwargs.pop('verbose', False))
+           log_to_console=kwargs.pop('verbose', False), log_route=section_name)
     try:
         klass(conf).run(once=once, **kwargs)
     except KeyboardInterrupt:

swift/common/db_replicator.py

@@ -92,7 +92,7 @@ class Replicator(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='replicator')
         self.root = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/common/middleware/catch_errors.py

@@ -30,7 +30,7 @@ class CatchErrorMiddleware(object):
         self.logger = getattr(app, 'logger', None)
         if not self.logger:
             # and only call get_logger if we have to
-            self.logger = get_logger(conf)
+            self.logger = get_logger(conf, log_route='catch-errors')
 
     def __call__(self, env, start_response):
         try:

swift/common/middleware/cname_lookup.py

@@ -53,7 +53,7 @@ class CNAMELookupMiddleware(object):
             self.storage_domain = '.' + self.storage_domain
         self.lookup_depth = int(conf.get('lookup_depth', '1'))
         self.memcache = None
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='cname-lookup')
 
     def __call__(self, env, start_response):
         if not self.storage_domain:

swift/common/middleware/ratelimit.py

@@ -39,7 +39,7 @@ class RateLimitMiddleware(object):
         if logger:
             self.logger = logger
         else:
-            self.logger = get_logger(conf)
+            self.logger = get_logger(conf, log_route='ratelimit')
         self.account_ratelimit = float(conf.get('account_ratelimit', 0))
         self.max_sleep_time_seconds = \
             float(conf.get('max_sleep_time_seconds', 60))

swift/common/middleware/swauth.py

@@ -51,7 +51,7 @@ class Swauth(object):
     def __init__(self, app, conf):
         self.app = app
         self.conf = conf
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='swauth')
         self.log_headers = conf.get('log_headers') == 'True'
         self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
         if self.reseller_prefix and self.reseller_prefix[-1] != '_':

swift/common/utils.py

@@ -395,6 +395,8 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None):
     :param conf: Configuration dict to read settings from
     :param name: Name of the logger
     :param log_to_console: Add handler which writes to console on stderr
+    :param log_route: Route for the logging, not emitted to the log, just used
+                      to separate logging configurations
     """
     if not conf:
         conf = {}

swift/common/wsgi.py

@@ -113,7 +113,7 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
         logger = kwargs.pop('logger')
     else:
         logger = get_logger(conf, log_name,
-                            log_to_console=kwargs.pop('verbose', False))
+            log_to_console=kwargs.pop('verbose', False), log_route='wsgi')
 
     # redirect errors to logger and close stdio
     capture_stdio(logger)

swift/container/auditor.py

@@ -28,7 +28,7 @@ class ContainerAuditor(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf, 'container-auditor')
+        self.logger = get_logger(conf, log_route='container-auditor')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/container/server.py

@@ -49,7 +49,7 @@ class ContainerController(object):
     save_headers = ['x-container-read', 'x-container-write']
 
     def __init__(self, conf):
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='container-server')
         self.root = conf.get('devices', '/srv/node/')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/container/updater.py

@@ -37,7 +37,7 @@ class ContainerUpdater(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf, 'container-updater')
+        self.logger = get_logger(conf, log_route='container-updater')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/obj/auditor.py

@@ -31,7 +31,7 @@ class ObjectAuditor(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf, 'object-auditor')
+        self.logger = get_logger(conf, log_route='object-auditor')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/obj/replicator.py

@@ -207,7 +207,7 @@ class ObjectReplicator(Daemon):
         :param logger: logging object
         """
         self.conf = conf
-        self.logger = get_logger(conf, 'object-replicator')
+        self.logger = get_logger(conf, log_route='object-replicator')
         self.devices_dir = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/obj/server.py

@@ -266,7 +266,7 @@ class ObjectController(object):
         <source-dir>/etc/object-server.conf-sample or
         /etc/swift/object-server.conf-sample.
         """
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='object-server')
         self.devices = conf.get('devices', '/srv/node/')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/obj/updater.py

@@ -35,7 +35,7 @@ class ObjectUpdater(Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = get_logger(conf, 'object-updater')
+        self.logger = get_logger(conf, log_route='object-updater')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

swift/proxy/server.py

@@ -1607,7 +1607,7 @@ class BaseApplication(object):
     def __init__(self, conf, memcache=None, logger=None, account_ring=None,
                  container_ring=None, object_ring=None):
         if logger is None:
-            self.logger = get_logger(conf)
+            self.logger = get_logger(conf, log_route='proxy-server')
         else:
             self.logger = logger
         if conf is None:

swift/stats/access_processor.py

@@ -34,7 +34,7 @@ class AccessLogProcessor(object):
                             conf.get('service_ips', '').split(',')\
                             if x.strip()]
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='access-processor')
 
     def log_line_parser(self, raw_log):
         '''given a raw access log line, return a dict of the good parts'''

swift/stats/account_stats.py

@@ -48,7 +48,8 @@ class AccountStat(Daemon):
         self.devices = server_conf.get('devices', '/srv/node')
         self.mount_check = server_conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')
-        self.logger = get_logger(stats_conf, 'swift-account-stats-logger')
+        self.logger = \
+            get_logger(stats_conf, log_route='account-stats')
 
     def run_once(self):
         self.logger.info(_("Gathering account stats"))

swift/stats/log_processor.py

@@ -40,7 +40,7 @@ class LogProcessor(object):
 
     def __init__(self, conf, logger):
         if isinstance(logger, tuple):
-            self.logger = get_logger(*logger)
+            self.logger = get_logger(*logger, log_route='log-processor')
         else:
             self.logger = logger
 
@@ -226,7 +226,7 @@ class LogProcessorDaemon(Daemon):
         c = conf.get('log-processor')
         super(LogProcessorDaemon, self).__init__(c)
         self.total_conf = conf
-        self.logger = get_logger(c)
+        self.logger = get_logger(c, log_route='log-processor')
         self.log_processor = LogProcessor(conf, self.logger)
         self.lookback_hours = int(c.get('lookback_hours', '120'))
         self.lookback_window = int(c.get('lookback_window',

swift/stats/log_uploader.py

@@ -65,7 +65,8 @@ class LogUploader(Daemon):
         self.filename_format = source_filename_format
         self.internal_proxy = InternalProxy(proxy_server_conf)
         log_name = 'swift-log-uploader-%s' % plugin_name
-        self.logger = utils.get_logger(uploader_conf, plugin_name)
+        self.logger = \
+            utils.get_logger(uploader_conf, plugin_name, log_route=plugin_name)
 
     def run_once(self):
         self.logger.info(_("Uploading logs"))

swift/stats/stats_processor.py

@@ -20,7 +20,7 @@ class StatsLogProcessor(object):
     """Transform account storage stat logs"""
 
     def __init__(self, conf):
-        self.logger = get_logger(conf)
+        self.logger = get_logger(conf, log_route='stats-processor')
 
     def process(self, obj_stream, data_object_account, data_object_container,
                 data_object_name):

test/unit/auth/test_server.py

@@ -456,7 +456,7 @@ class TestAuthServer(unittest.TestCase):
     def test_basic_logging(self):
         log = StringIO()
         log_handler = StreamHandler(log)
-        logger = get_logger(self.conf, 'auth')
+        logger = get_logger(self.conf, 'auth-server', log_route='auth-server')
         logger.logger.addHandler(log_handler)
         try:
             auth_server.http_connect = fake_http_connect(201)
@@ -534,7 +534,7 @@ class TestAuthServer(unittest.TestCase):
         orig_Request = auth_server.Request
         log = StringIO()
         log_handler = StreamHandler(log)
-        logger = get_logger(self.conf, 'auth')
+        logger = get_logger(self.conf, 'auth-server', log_route='auth-server')
         logger.logger.addHandler(log_handler)
         try:
             auth_server.Request = request_causing_exception

test/unit/common/test_daemon.py

@@ -28,7 +28,7 @@ class MyDaemon(daemon.Daemon):
 
     def __init__(self, conf):
         self.conf = conf
-        self.logger = utils.get_logger(None, 'server')
+        self.logger = utils.get_logger(None, 'server', log_route='server')
         MyDaemon.forever_called = False
         MyDaemon.once_called = False
 
@@ -99,7 +99,7 @@ user = %s
             sio = StringIO()
             logger = logging.getLogger('server')
             logger.addHandler(logging.StreamHandler(sio))
-            logger = utils.get_logger(None, 'server')
+            logger = utils.get_logger(None, 'server', log_route='server')
             daemon.run_daemon(MyDaemon, conf_file, logger=logger)
             self.assert_('user quit' in sio.getvalue().lower())
 

test/unit/common/test_utils.py

@@ -289,17 +289,19 @@ Error: unable to locate %s
         sio = StringIO()
         logger = logging.getLogger('server')
         logger.addHandler(logging.StreamHandler(sio))
-        logger = utils.get_logger(None, 'server')
+        logger = utils.get_logger(None, 'server', log_route='server')
         logger.warn('test1')
         self.assertEquals(sio.getvalue(), 'test1\n')
         logger.debug('test2')
         self.assertEquals(sio.getvalue(), 'test1\n')
-        logger = utils.get_logger({'log_level': 'DEBUG'}, 'server')
+        logger = utils.get_logger({'log_level': 'DEBUG'}, 'server',
+                                  log_route='server')
         logger.debug('test3')
         self.assertEquals(sio.getvalue(), 'test1\ntest3\n')
         # Doesn't really test that the log facility is truly being used all the
         # way to syslog; but exercises the code.
-        logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server')
+        logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server',
+                                  log_route='server')
         logger.warn('test4')
         self.assertEquals(sio.getvalue(),
                           'test1\ntest3\ntest4\n')