From 2c596c0a0fbb0a1ec74092278d9f8844b90bf760 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Fri, 20 Aug 2010 00:42:38 +0000 Subject: [PATCH] Initial commit of middleware refactor --- bin/swift-account-auditor | 9 ++-- bin/swift-account-reaper | 9 ++-- bin/swift-account-replicator | 9 ++-- bin/swift-account-server | 11 ++-- bin/swift-auth-server | 11 ++-- bin/swift-container-auditor | 9 ++-- bin/swift-container-replicator | 9 ++-- bin/swift-container-server | 11 ++-- bin/swift-container-updater | 7 ++- bin/swift-object-auditor | 9 ++-- bin/swift-object-replicator | 39 +++++--------- bin/swift-object-server | 12 ++--- bin/swift-object-updater | 7 ++- bin/swift-proxy-server | 32 ++---------- etc/account-server.conf-sample | 23 ++++++--- etc/auth-server.conf-sample | 17 +++++-- etc/container-server.conf-sample | 28 +++++----- etc/object-server.conf-sample | 34 +++++++------ etc/proxy-server.conf-sample | 38 +++++++++----- setup.py | 51 ++++++++++++------- swift/account/auditor.py | 18 +++---- swift/account/reaper.py | 20 ++++---- swift/account/server.py | 16 +++--- swift/auth/server.py | 9 ++-- swift/common/db_replicator.py | 26 +++++----- swift/common/healthcheck.py | 29 ----------- swift/common/middleware/__init__.py | 0 swift/common/{ => middleware}/auth.py | 45 +++++++++------- swift/common/middleware/cache.py | 34 +++++++++++++ swift/common/middleware/healthcheck.py | 43 ++++++++++++++++ swift/common/utils.py | 13 ++++- swift/common/wsgi.py | 18 ++++--- swift/container/auditor.py | 18 +++---- swift/container/server.py | 17 ++++--- swift/container/updater.py | 20 ++++---- swift/obj/auditor.py | 16 +++--- swift/obj/replicator.py | 2 +- swift/obj/server.py | 19 +++---- swift/obj/updater.py | 20 ++++---- swift/proxy/server.py | 33 +++++------- test/unit/account/test_server.py | 27 ---------- test/unit/auth/test_server.py | 2 +- test/unit/common/middleware/__init__.py | 0 .../unit/common/{ => middleware}/test_auth.py | 2 +- test/unit/common/middleware/test_cache.py | 42 +++++++++++++++ .../{ => middleware}/test_healthcheck.py | 24 ++++++--- test/unit/common/test_db_replicator.py | 16 +++--- test/unit/container/test_server.py | 27 ---------- test/unit/container/test_updater.py | 36 ++++++++----- test/unit/obj/test_server.py | 27 ---------- test/unit/obj/test_updater.py | 24 ++++++--- test/unit/proxy/test_server.py | 20 -------- 52 files changed, 535 insertions(+), 503 deletions(-) delete mode 100644 swift/common/healthcheck.py create mode 100644 swift/common/middleware/__init__.py rename swift/common/{ => middleware}/auth.py (68%) create mode 100644 swift/common/middleware/cache.py create mode 100644 swift/common/middleware/healthcheck.py create mode 100644 test/unit/common/middleware/__init__.py rename test/unit/common/{ => middleware}/test_auth.py (99%) create mode 100644 test/unit/common/middleware/test_cache.py rename test/unit/common/{ => middleware}/test_healthcheck.py (54%) diff --git a/bin/swift-account-auditor b/bin/swift-account-auditor index 2bbb73c23f..cba89e9e63 100755 --- a/bin/swift-account-auditor +++ b/bin/swift-account-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('account-server')) if c.has_section('account-auditor'): - auditor_conf = dict(c.items('account-auditor')) + conf = dict(c.items('account-auditor')) else: print "Unable to find account-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'account-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = AccountAuditor(server_conf, auditor_conf) + auditor = AccountAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-account-reaper b/bin/swift-account-reaper index d62e43add8..c5619d0785 100755 --- a/bin/swift-account-reaper +++ b/bin/swift-account-reaper @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('account-server')) if c.has_section('account-reaper'): - reaper_conf = dict(c.items('account-reaper')) + conf = dict(c.items('account-reaper')) else: print "Unable to find account-reaper config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(reaper_conf, 'account-reaper') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - reaper = AccountReaper(server_conf, reaper_conf) + reaper = AccountReaper(conf) if once: reaper.reap_once() else: diff --git a/bin/swift-account-replicator b/bin/swift-account-replicator index 3e47eaa4f1..284c751598 100755 --- a/bin/swift-account-replicator +++ b/bin/swift-account-replicator @@ -41,17 +41,16 @@ if __name__ == '__main__': sys.exit(1) once = len(args) > 1 and args[1] == 'once' - server_conf = dict(c.items('account-server')) if c.has_section('account-replicator'): - replicator_conf = dict(c.items('account-replicator')) + conf = dict(c.items('account-replicator')) else: print "Unable to find account-replicator config section in %s." % \ args[0] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: - AccountReplicator(server_conf, replicator_conf).replicate_once() + AccountReplicator(conf).replicate_once() else: - AccountReplicator(server_conf, replicator_conf).replicate_forever() + AccountReplicator(conf).replicate_forever() diff --git a/bin/swift-account-server b/bin/swift-account-server index d5e8021049..6be5706ea4 100755 --- a/bin/swift-account-server +++ b/bin/swift-account-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.account.server import AccountController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('account-server')) - run_wsgi(AccountController, conf, default_port=6002) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6002) diff --git a/bin/swift-auth-server b/bin/swift-auth-server index 775530e078..5f32912a9c 100755 --- a/bin/swift-auth-server +++ b/bin/swift-auth-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.auth.server import AuthController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('auth-server')) - run_wsgi(AuthController, conf, default_port=11000) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=11000) diff --git a/bin/swift-container-auditor b/bin/swift-container-auditor index 1ff1682c52..6dfdc4592d 100755 --- a/bin/swift-container-auditor +++ b/bin/swift-container-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('container-server')) if c.has_section('container-auditor'): - auditor_conf = dict(c.items('container-auditor')) + conf = dict(c.items('container-auditor')) else: print "Unable to find container-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'container-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(onf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = ContainerAuditor(server_conf, auditor_conf) + auditor = ContainerAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-container-replicator b/bin/swift-container-replicator index db2ffc8b20..663cb51db7 100755 --- a/bin/swift-container-replicator +++ b/bin/swift-container-replicator @@ -41,17 +41,16 @@ if __name__ == '__main__': sys.exit(1) once = len(args) > 1 and args[1] == 'once' - server_conf = dict(c.items('container-server')) if c.has_section('container-replicator'): - replicator_conf = dict(c.items('container-replicator')) + conf = dict(c.items('container-replicator')) else: print "Unable to find container-replicator config section in %s." % \ args[0] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) if once or '--once' in [opt[0] for opt in optlist]: - ContainerReplicator(server_conf, replicator_conf).replicate_once() + ContainerReplicator(conf).replicate_once() else: - ContainerReplicator(server_conf, replicator_conf).replicate_forever() + ContainerReplicator(conf).replicate_forever() diff --git a/bin/swift-container-server b/bin/swift-container-server index e0ad813f46..f707eb091c 100755 --- a/bin/swift-container-server +++ b/bin/swift-container-server @@ -14,17 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.container.server import ContainerController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('container-server')) - run_wsgi(ContainerController, conf, default_port=6001) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6001) diff --git a/bin/swift-container-updater b/bin/swift-container-updater index c37ba3680a..7c23115123 100755 --- a/bin/swift-container-updater +++ b/bin/swift-container-updater @@ -34,15 +34,14 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('container-server')) if c.has_section('container-updater'): - updater_conf = dict(c.items('container-updater')) + conf = dict(c.items('container-updater')) else: print "Unable to find container-updater config section in %s." % \ sys.argv[1] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -56,7 +55,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - updater = ContainerUpdater(server_conf, updater_conf) + updater = ContainerUpdater(conf) if once: updater.update_once_single_threaded() else: diff --git a/bin/swift-object-auditor b/bin/swift-object-auditor index 5d0e54e661..e995ad8725 100755 --- a/bin/swift-object-auditor +++ b/bin/swift-object-auditor @@ -34,21 +34,20 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('object-server')) if c.has_section('object-auditor'): - auditor_conf = dict(c.items('object-auditor')) + conf = dict(c.items('object-auditor')) else: print "Unable to find object-auditor config section in %s." % \ sys.argv[1] sys.exit(1) - logger = utils.get_logger(auditor_conf, 'object-auditor') + logger = utils.get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = utils.LoggerFileObject(logger) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -62,7 +61,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - auditor = ObjectAuditor(server_conf, auditor_conf) + auditor = ObjectAuditor(conf) if once: auditor.audit_once() else: diff --git a/bin/swift-object-replicator b/bin/swift-object-replicator index 0de0b5d64a..6a2da4b4f3 100755 --- a/bin/swift-object-replicator +++ b/bin/swift-object-replicator @@ -27,38 +27,23 @@ from swift.common.utils import get_logger, drop_privileges, LoggerFileObject TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) -def read_configs(conf_file): - c = ConfigParser() - if not c.read(conf_file): - print "Unable to read config file: %s" % conf_file - sys.exit(1) - conf = dict(c.items('object-server')) - repl_conf = dict(c.items('object-replicator')) - if not repl_conf: - sys.exit() - conf['replication_concurrency'] = repl_conf.get('concurrency',1) - conf['vm_test_mode'] = repl_conf.get('vm_test_mode', 'no') - conf['daemonize'] = repl_conf.get('daemonize', 'yes') - conf['run_pause'] = repl_conf.get('run_pause', '30') - conf['log_facility'] = repl_conf.get('log_facility', 'LOG_LOCAL1') - conf['log_level'] = repl_conf.get('log_level', 'INFO') - conf['timeout'] = repl_conf.get('timeout', '5') - conf['stats_interval'] = repl_conf.get('stats_interval', '3600') - conf['reclaim_age'] = int(repl_conf.get('reclaim_age', 86400)) - - return conf - if __name__ == '__main__': if len(sys.argv) < 2: print "Usage: object-replicator CONFIG_FILE [once]" sys.exit() - try: - conf = read_configs(sys.argv[1]) - except: - print "Problem reading the config. Aborting object replication." - sys.exit() + c = ConfigParser() + if not c.read(conf_file): + print "Unable to read config file: %s" % conf_file + sys.exit(1) + if c.has_section('object-replicator'): + conf = dict(c.items('object-replicator')) + else: + print "Unable to find object-replicator config section in %s" % \ + sys.argv[1] + sys.exit(1) + once = len(sys.argv) > 2 and sys.argv[2] == 'once' - logger = get_logger(conf, 'object-replicator') + logger = get_logger(conf) # log uncaught exceptions sys.excepthook = lambda *exc_info: \ logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) diff --git a/bin/swift-object-server b/bin/swift-object-server index c0603b96a7..ce74b17195 100755 --- a/bin/swift-object-server +++ b/bin/swift-object-server @@ -14,17 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser import sys from swift.common.wsgi import run_wsgi -from swift.obj.server import ObjectController if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('object-server')) - run_wsgi(ObjectController, conf, default_port=6000) - + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=6000) diff --git a/bin/swift-object-updater b/bin/swift-object-updater index 595840a079..a9512a4e76 100755 --- a/bin/swift-object-updater +++ b/bin/swift-object-updater @@ -34,15 +34,14 @@ if __name__ == '__main__': print "Unable to read config file." sys.exit(1) - server_conf = dict(c.items('object-server')) if c.has_section('object-updater'): - updater_conf = dict(c.items('object-updater')) + conf = dict(c.items('object-updater')) else: print "Unable to find object-updater config section in %s." % \ sys.argv[1] sys.exit(1) - utils.drop_privileges(server_conf.get('user', 'swift')) + utils.drop_privileges(conf.get('user', 'swift')) try: os.setsid() @@ -56,7 +55,7 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, kill_children) - updater = ObjectUpdater(server_conf, updater_conf) + updater = ObjectUpdater(conf) if once: updater.update_once_single_threaded() else: diff --git a/bin/swift-proxy-server b/bin/swift-proxy-server index 0ba14e2357..c56df2e074 100755 --- a/bin/swift-proxy-server +++ b/bin/swift-proxy-server @@ -14,37 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ConfigParser import ConfigParser -import os import sys from swift.common.wsgi import run_wsgi -from swift.common.auth import DevAuthMiddleware -from swift.common.memcached import MemcacheRing -from swift.common.utils import get_logger -from swift.proxy.server import Application if __name__ == '__main__': - c = ConfigParser() - if not c.read(sys.argv[1]): - print "Unable to read config file." - sys.exit(1) - conf = dict(c.items('proxy-server')) - if c.has_section('auth-server'): - auth_conf = dict(c.items('auth-server')) - else: - auth_conf = {} - swift_dir = conf.get('swift_dir', '/etc/swift') - m, c = auth_conf.get('class', - 'swift.common.auth.DevAuthMiddleware').rsplit('.', 1) - m = __import__(m, fromlist=[c]) - authware = m.__dict__[c] - - memcache = MemcacheRing([s.strip() for s in - conf.get('memcache_servers', '127.0.0.1:11211').split(',') - if s.strip()]) - logger = get_logger(conf, 'proxy') - app = Application(conf, memcache, logger) - # Wrap the app with auth - app = authware(app, auth_conf, memcache, logger) - run_wsgi(app, conf, logger=logger, default_port=80) + if len(sys.argv) != 2: + print "Usage: %s CONFIG_FILE" + run_wsgi(sys.argv[1], default_port=80) diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 3ded523ee2..303ab0e00b 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -1,15 +1,23 @@ -[account-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = account +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6002 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:swift#account [account-replicator] +log_name = account-replicator # log_facility = LOG_LOCAL0 # log_level = INFO # per_diff = 1000 @@ -26,6 +34,7 @@ # reclaim_age = 86400 [account-stats] +log_name = account-stats # cf_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 # container_name = account_stats # proxy_server_conf = /etc/swift/proxy-server.conf @@ -33,6 +42,7 @@ # log_level = INFO [account-auditor] +log_name = account-auditor # Will audit, at most, 1 account per device per interval # interval = 1800 # Maximum containers randomly picked for a given account audit @@ -43,6 +53,7 @@ # log_level = INFO [account-reaper] +log_name = account-reaper # concurrency = 25 # interval = 3600 # node_timeout = 10 diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample index 29bcc63f1a..61c1ffafd9 100644 --- a/etc/auth-server.conf-sample +++ b/etc/auth-server.conf-sample @@ -1,15 +1,22 @@ -[auth-server] -# swift_dir = /etc/swift +[DEFAULT] +log_name = auth # bind_ip = 0.0.0.0 # bind_port = 11000 # log_facility = LOG_LOCAL0 # log_level = INFO # workers = 1 +# user = swift +# swift_dir = /etc/swift +# cert_file = Default is no cert; format is path like /etc/swift/auth.crt +# key_file = Default is no key; format is path like /etc/swift/auth.key + +[pipeline:main] +pipeline = auth-server + +[app:auth-server] +use = egg:swift#auth # reseller_prefix = AUTH # default_cluster_url = http://127.0.0.1:9000/v1 # token_life = 86400 # log_headers = False -# cert_file = Default is no cert; format is path like /etc/swift/auth.crt -# key_file = Default is no key; format is path like /etc/swift/auth.key # node_timeout = 10 -user = swift diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 086d4d8a0f..ccad08c710 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -1,19 +1,25 @@ -[container-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = container +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6001 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:swift#container # node_timeout = 3 # conn_timeout = 0.5 [container-replicator] -# log_facility = LOG_LOCAL0 -# log_level = INFO +log_name = container-replicator # per_diff = 1000 # concurrency = 8 # run_pause = 30 @@ -23,21 +29,19 @@ # reclaim_age = 604800 [container-updater] +log_name = container-updater # interval = 300 # concurrency = 4 # node_timeout = 3 # conn_timeout = 0.5 # slowdown will sleep that amount between containers # slowdown = 0.01 -# log_facility = LOG_LOCAL0 -# log_level = INFO [container-auditor] +log_name = container-auditor # Will audit, at most, 1 container per device per interval # interval = 1800 # Maximum objects randomly picked for a given container audit # max_object_count = 100 # node_timeout = 10 # conn_timeout = 0.5 -# log_facility = LOG_LOCAL0 -# log_level = INFO diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index cce9ce39a1..81a35c5cb7 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -1,24 +1,30 @@ -[object-server] -# swift_dir = /etc/swift -# devices = /srv/node -# mount_check = true +[DEFAULT] +log_name = object +# log_facility = LOG_LOCAL0 +# log_level = INFO # bind_ip = 0.0.0.0 # bind_port = 6000 # workers = 1 -# log_facility = LOG_LOCAL0 -# log_level = INFO -# log_requests = True # user = swift +# swift_dir = /etc/swift +# devices = /srv/node +# mount_check = true + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:swift#object +# log_requests = True # node_timeout = 3 # conn_timeout = 0.5 -# network_chunk_size = 8192 -# disk_chunk_size = 32768 +# network_chunk_size = 65536 +# disk_chunk_size = 65536 # max_upload_time = 86400 # slow = 1 [object-replicator] -# log_facility = LOG_LOCAL0 -# log_level = INFO +log_name = object-replicator # daemonize = on # run_pause = 30 # concurrency = 1 @@ -28,19 +34,17 @@ # reclaim_age = 604800 [object-updater] +log_name = object-updater # interval = 300 # concurrency = 1 # node_timeout = 10 # conn_timeout = 0.5 # slowdown will sleep that amount between objects # slowdown = 0.01 -# log_facility = LOG_LOCAL0 -# log_level = INFO [object-auditor] +log_name = object-auditor # Will audit, at most, 1 object per device per interval # interval = 1800 # node_timeout = 10 # conn_timeout = 0.5 -# log_facility = LOG_LOCAL0 -# log_level = INFO diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 33d612c724..d8bbc8f76b 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -1,21 +1,26 @@ -[proxy-server] -# bind_ip = 0.0.0.0 -# bind_port = 80 -# cert_file = /etc/swift/proxy.crt -# key_file = /etc/swift/proxy.key -# swift_dir = /etc/swift +[DEFAULT] +log_name = proxy # log_facility = LOG_LOCAL0 # log_level = INFO -# log_headers = False +# bind_ip = 0.0.0.0 +# bind_port = 80 # workers = 1 # user = swift +# cert_file = /etc/swift/proxy.crt +# key_file = /etc/swift/proxy.key + +[pipeline:main] +pipeline = healthcheck cache auth proxy + +[app:proxy] +use = egg:swift#proxy +filter-with = auth +# swift_dir = /etc/swift +# log_headers = False # recheck_account_existence = 60 # recheck_container_existence = 60 # object_chunk_size = 8192 # client_chunk_size = 8192 -# Default for memcache_servers is below, but you can specify multiple servers -# with the format: 10.1.2.3:11211,10.1.2.4:11211 -# memcache_servers = 127.0.0.1:11211 # node_timeout = 10 # client_timeout = 60 # conn_timeout = 0.5 @@ -31,8 +36,17 @@ # rate_limit_account_whitelist = acct1,acct2,etc # rate_limit_account_blacklist = acct3,acct4,etc -# [auth-server] -# class = swift.common.auth.DevAuthMiddleware +[filter:auth] +use = egg:swift#auth # ip = 127.0.0.1 # port = 11000 # node_timeout = 10 + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#cache +# Default for memcache_servers is below, but you can specify multiple servers +# with the format: 10.1.2.3:11211,10.1.2.4:11211 +# memcache_servers = 127.0.0.1:11211 diff --git a/setup.py b/setup.py index 1c711b764c..02dd389495 100644 --- a/setup.py +++ b/setup.py @@ -55,21 +55,38 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 2.6', 'Environment :: No Input/Output (Daemon)', - ], + ], install_requires=[], # removed for better compat - scripts=['bin/st', 'bin/swift-account-auditor', - 'bin/swift-account-audit', 'bin/swift-account-reaper', - 'bin/swift-account-replicator', 'bin/swift-account-server', - 'bin/swift-auth-create-account', - 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', - 'bin/swift-container-auditor', - 'bin/swift-container-replicator', - 'bin/swift-container-server', 'bin/swift-container-updater', - 'bin/swift-drive-audit', 'bin/swift-get-nodes', - 'bin/swift-init', 'bin/swift-object-auditor', - 'bin/swift-object-info', - 'bin/swift-object-replicator', - 'bin/swift-object-server', - 'bin/swift-object-updater', 'bin/swift-proxy-server', - 'bin/swift-ring-builder', 'bin/swift-stats-populate', - 'bin/swift-stats-report']) + scripts=[ + 'bin/st', 'bin/swift-account-auditor', + 'bin/swift-account-audit', 'bin/swift-account-reaper', + 'bin/swift-account-replicator', 'bin/swift-account-server', + 'bin/swift-auth-create-account', + 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', + 'bin/swift-container-auditor', + 'bin/swift-container-replicator', + 'bin/swift-container-server', 'bin/swift-container-updater', + 'bin/swift-drive-audit', 'bin/swift-get-nodes', + 'bin/swift-init', 'bin/swift-object-auditor', + 'bin/swift-object-info', + 'bin/swift-object-replicator', + 'bin/swift-object-server', + 'bin/swift-object-updater', 'bin/swift-proxy-server', + 'bin/swift-ring-builder', 'bin/swift-stats-populate', + 'bin/swift-stats-report' + ], + entry_points={ + 'paste.app_factory' : [ + 'proxy=swift.proxy.server:app_factory', + 'object=swift.obj.server:app_factory', + 'container=swift.container.server:app_factory', + 'account=swift.account.server:app_factory', + 'auth=swift.auth.server:app_factory', + ], + 'paste.filter_factory' : [ + 'auth=swift.common.middleware.auth:filter_factory', + 'healthcheck=swift.common.middleware.healthcheck:filter_factory', + 'cache=swift.common.middleware.cache:filter_factory', + ], + }, + ) diff --git a/swift/account/auditor.py b/swift/account/auditor.py index a21ed0849b..249d9f436d 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -35,19 +35,19 @@ class AuditException(Exception): class AccountAuditor(object): """Audit accounts.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'account-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf, 'account-auditor') + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.max_container_count = \ - int(auditor_conf.get('max_container_count', 100)) + int(conf.get('max_container_count', 100)) self.container_passes = 0 self.container_failures = 0 self.container_errors = 0 diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 93b54e7525..f747e3ce09 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -50,25 +50,23 @@ class AccountReaper(object): configuration parameters. """ - log_name = 'account-reaper' - - def __init__(self, server_conf, reaper_conf): - self.logger = get_logger(reaper_conf, self.log_name) - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(reaper_conf.get('interval', 3600)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 3600)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.account_ring = None self.container_ring = None self.object_ring = None - self.node_timeout = int(reaper_conf.get('node_timeout', 10)) - self.conn_timeout = float(reaper_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.myips = whataremyips() - self.concurrency = int(reaper_conf.get('concurrency', 25)) + self.concurrency = int(conf.get('concurrency', 25)) self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency) diff --git a/swift/account/server.py b/swift/account/server.py index 6e5dd7472e..7194cc9f95 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -31,10 +31,9 @@ from xml.sax import saxutils from swift.common.db import AccountBroker from swift.common.utils import get_param, split_path, storage_directory, \ - hash_path + hash_path, get_logger from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_xml_encodable -from swift.common.healthcheck import healthcheck from swift.common.db_replicator import ReplicatorRpc @@ -43,10 +42,9 @@ DATADIR = 'accounts' class AccountController(object): """WSGI controller for the account server.""" - log_name = 'account' def __init__(self, conf): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -253,9 +251,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -288,3 +284,9 @@ class AccountController(object): else: self.logger.info(log_message) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI account server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return AccountController(conf) diff --git a/swift/auth/server.py b/swift/auth/server.py index 676b6fc6d9..48b40ab50f 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -89,10 +89,8 @@ class AuthController(object): configuration parameters. """ - log_name = 'auth' - def __init__(self, conf, ring=None): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.default_cluster_url = \ conf.get('default_cluster_url', 'http://127.0.0.1:9000/v1') @@ -500,3 +498,8 @@ class AuthController(object): def __call__(self, env, start_response): """ Used by the eventlet.wsgi.server """ return self.handleREST(env, start_response) + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return AuthController(conf) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 5f416cce31..ce66a95fb0 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -89,28 +89,28 @@ class Replicator(object): Implements the logic for directing db replication. """ - def __init__(self, server_conf, replicator_conf): + def __init__(self, conf): self.logger = \ - get_logger(replicator_conf, '%s-replicator' % self.server_type) + get_logger(conf) # log uncaught exceptions sys.excepthook = lambda * exc_info: \ self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) sys.stdout = sys.stderr = LoggerFileObject(self.logger) - self.root = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + self.root = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.port = int(server_conf.get('bind_port', self.default_port)) - concurrency = int(replicator_conf.get('concurrency', 8)) + self.port = int(conf.get('bind_port', self.default_port)) + concurrency = int(conf.get('concurrency', 8)) self.cpool = GreenPool(size=concurrency) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + swift_dir = conf.get('swift_dir', '/etc/swift') self.ring = ring.Ring(os.path.join(swift_dir, self.ring_file)) - self.per_diff = int(replicator_conf.get('per_diff', 1000)) - self.run_pause = int(replicator_conf.get('run_pause', 30)) - self.vm_test_mode = replicator_conf.get( + self.per_diff = int(conf.get('per_diff', 1000)) + self.run_pause = int(conf.get('run_pause', 30)) + self.vm_test_mode = conf.get( 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') - self.node_timeout = int(replicator_conf.get('node_timeout', 10)) - self.conn_timeout = float(replicator_conf.get('conn_timeout', 0.5)) - self.reclaim_age = float(replicator_conf.get('reclaim_age', 86400 * 7)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7)) self._zero_stats() def _zero_stats(self): diff --git a/swift/common/healthcheck.py b/swift/common/healthcheck.py deleted file mode 100644 index 0e18f240a2..0000000000 --- a/swift/common/healthcheck.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from webob import Response - - -class HealthCheckController(object): - """Basic controller used for monitoring.""" - - def __init__(self, *args, **kwargs): - pass - - @classmethod - def GET(self, req): - return Response(request=req, body="OK", content_type="text/plain") - -healthcheck = HealthCheckController.GET diff --git a/swift/common/middleware/__init__.py b/swift/common/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/swift/common/auth.py b/swift/common/middleware/auth.py similarity index 68% rename from swift/common/auth.py rename to swift/common/middleware/auth.py index 6830b60ba7..13d0e426a2 100644 --- a/swift/common/auth.py +++ b/swift/common/middleware/auth.py @@ -21,6 +21,8 @@ from eventlet.timeout import Timeout from swift.common.utils import split_path from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.utils import get_logger, cache_from_env +from swift.common.memcached import MemcacheRing class DevAuthMiddleware(object): @@ -28,33 +30,37 @@ class DevAuthMiddleware(object): Auth Middleware that uses the dev auth server """ - def __init__(self, app, conf, memcache_client, logger): + def __init__(self, app, conf, memcache_client=None, logger=None): self.app = app self.memcache_client = memcache_client - self.logger = logger + if logger is None: + self.logger = get_logger(conf) + else: + self.logger = logger self.conf = conf self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) self.timeout = int(conf.get('node_timeout', 10)) def __call__(self, env, start_response): + if self.memcache_client is None: + self.memcache_client = cache_from_env(env) req = Request(env) - if req.path != '/healthcheck': - if 'x-storage-token' in req.headers and \ - 'x-auth-token' not in req.headers: - req.headers['x-auth-token'] = req.headers['x-storage-token'] + if 'x-storage-token' in req.headers and \ + 'x-auth-token' not in req.headers: + req.headers['x-auth-token'] = req.headers['x-storage-token'] + try: version, account, container, obj = split_path(req.path, 1, 4, True) - if account is None: - return HTTPPreconditionFailed(request=req, body='Bad URL')( - env, start_response) - if not req.headers.get('x-auth-token'): - return HTTPPreconditionFailed(request=req, - body='Missing Auth Token')(env, start_response) - if account is None: - return HTTPPreconditionFailed( - request=req, body='Bad URL')(env, start_response) - if not self.auth(account, req.headers['x-auth-token']): - return HTTPUnauthorized(request=req)(env, start_response) + except ValueError, e: + version = account = container = obj = None + if account is None: + return HTTPPreconditionFailed(request=req, body='Bad URL')( + env, start_response) + if not req.headers.get('x-auth-token'): + return HTTPPreconditionFailed(request=req, + body='Missing Auth Token')(env, start_response) + if not self.auth(account, req.headers['x-auth-token']): + return HTTPUnauthorized(request=req)(env, start_response) # If we get here, then things should be good. return self.app(env, start_response) @@ -95,3 +101,8 @@ class DevAuthMiddleware(object): val = (now, validated) self.memcache_client.set(key, val, timeout=validated) return True + +def filter_factory(global_conf, **local_conf): + def auth_filter(app): + return DevAuthMiddleware(app, local_conf) + return auth_filter diff --git a/swift/common/middleware/cache.py b/swift/common/middleware/cache.py new file mode 100644 index 0000000000..a0dfc81b86 --- /dev/null +++ b/swift/common/middleware/cache.py @@ -0,0 +1,34 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common.memcached import MemcacheRing + +class CacheMiddleware(object): + """ + Caching middleware that manages caching in swift. + """ + def __init__(self, app, conf): + self.app = app + self.memcache = MemcacheRing([s.strip() for s in + conf.get('memcache_servers', '127.0.0.1:11211').split(',') + if s.strip()]) + + def __call__(self, env, start_response): + env['swift.cache'] = self.memcache + return self.app(env, start_response) + +def filter_factory(global_conf, **local_conf): + def cache_filter(app): + return CacheMiddleware(app, local_conf) + return cache_filter diff --git a/swift/common/middleware/healthcheck.py b/swift/common/middleware/healthcheck.py new file mode 100644 index 0000000000..3bfc745bc2 --- /dev/null +++ b/swift/common/middleware/healthcheck.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from webob import Request, Response + + +class HealthCheckMiddleware(object): + """ + Healthcheck middleware used for monitoring. + + If the path is /healthcheck, it will respond with "OK" in the body + """ + + def __init__(self, app, *args, **kwargs): + self.app = app + + def GET(self, req): + """Returns a 200 response with "OK" in the body.""" + return Response(request=req, body="OK", content_type="text/plain") + + def __call__(self, env, start_response): + req = Request(env) + if req.path == '/healthcheck': + return self.GET(req)(env, start_response) + else: + return self.app(env, start_response) + +def filter_factory(global_conf, **local_conf): + def healthcheck_filter(app): + return HealthCheckMiddleware(app) + return healthcheck_filter diff --git a/swift/common/utils.py b/swift/common/utils.py index dab0b9d255..e4962f2367 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -323,7 +323,7 @@ class NamedLogger(object): call('%s %s: %s' % (self.server, msg, emsg), *args) -def get_logger(conf, name): +def get_logger(conf, name=None): """ Get the current system logger using config settings. @@ -342,6 +342,8 @@ def get_logger(conf, name): if conf is None: root_logger.setLevel(logging.INFO) return NamedLogger(root_logger, name) + if name is None: + name = conf.get('log_name', 'swift') get_logger.handler = SysLogHandler(address='/dev/log', facility=getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0)) @@ -513,3 +515,12 @@ def unlink_older_than(path, mtime): os.unlink(fpath) except OSError: pass + +def item_from_env(env, item_name): + item = env.get(item_name, None) + if item is None: + logging.error("ERROR: %s could not be found in env!" % item_name) + return item + +def cache_from_env(env): + return item_from_env(env, 'swift.cache') diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index f0045e0393..2944bd3bdf 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -24,6 +24,7 @@ import mimetools import eventlet from eventlet import greenio, GreenPool, sleep, wsgi, listen +from paste.deploy import loadapp, appconfig # Hook to ensure connection resets don't blow up our servers. # Remove with next release of Eventlet that has it in the set already. @@ -58,18 +59,24 @@ def monkey_patch_mimetools(): # We might be able to pull pieces of this out to test, but right now it seems # like more work than it's worth. -def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover +def run_wsgi(conf_file, *args, **kwargs): # pragma: no cover """ Loads common settings from conf, then instantiates app and runs the server using the specified number of workers. - :param app: WSGI callable - :param conf: Configuration dictionary + :param conf_file: Path to paste.deploy style configuration file """ + + try: + app = loadapp('config:%s' % conf_file) + conf = appconfig('config:%s' % conf_file) + except Exception, e: + print "Error trying to load config %s: %s" % (conf_file, e) + return if 'logger' in kwargs: logger = kwargs['logger'] else: - logger = get_logger(conf, app.log_name) + logger = get_logger(conf) # log uncaught exceptions sys.excepthook = lambda * exc_info: \ @@ -103,9 +110,6 @@ def run_wsgi(app, conf, *args, **kwargs): # pragma: no cover sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) worker_count = int(conf.get('workers', '1')) drop_privileges(conf.get('user', 'swift')) - if isinstance(app, type): - # Instantiate app if it hasn't been already - app = app(conf, *args) def run_server(): wsgi.HttpProtocol.default_request_version = "HTTP/1.0" diff --git a/swift/container/auditor.py b/swift/container/auditor.py index b507b0c57d..a41bf5358c 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -36,20 +36,20 @@ class AuditException(Exception): class ContainerAuditor(object): """Audit containers.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'container-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring = None self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.object_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) - self.max_object_count = int(auditor_conf.get('max_object_count', 100)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) + self.max_object_count = int(conf.get('max_object_count', 100)) self.account_passes = 0 self.account_failures = 0 self.account_errors = 0 diff --git a/swift/container/server.py b/swift/container/server.py index 57775ee661..fab7275980 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -31,11 +31,10 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.db import ContainerBroker from swift.common.utils import get_logger, get_param, hash_path, \ - storage_directory, split_path + storage_directory, split_path, get_logger from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_xml_encodable from swift.common.bufferedhttp import http_connect -from swift.common.healthcheck import healthcheck from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc @@ -45,10 +44,8 @@ DATADIR = 'containers' class ContainerController(object): """WSGI Controller for the container server.""" - log_name = 'container' - def __init__(self, conf): - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -347,9 +344,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -378,3 +373,9 @@ class ContainerController(object): else: self.logger.info(log_message) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI container server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return ContainerController(conf) diff --git a/swift/container/updater.py b/swift/container/updater.py index a0db68cfa6..003d06b130 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -33,19 +33,19 @@ from swift.common.utils import get_logger, whataremyips class ContainerUpdater(object): """Update container information in account listings.""" - def __init__(self, server_conf, updater_conf): - self.logger = get_logger(updater_conf, 'container-updater') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf, 'container-updater') + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = server_conf.get('swift_dir', '/etc/swift') - self.interval = int(updater_conf.get('interval', 300)) + swift_dir = conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 300)) self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.account_ring = None - self.concurrency = int(updater_conf.get('concurrency', 4)) - self.slowdown = float(updater_conf.get('slowdown', 0.01)) - self.node_timeout = int(updater_conf.get('node_timeout', 3)) - self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.concurrency = int(conf.get('concurrency', 4)) + self.slowdown = float(conf.get('slowdown', 0.01)) + self.node_timeout = int(conf.get('node_timeout', 3)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.no_changes = 0 self.successes = 0 self.failures = 0 diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 62484ffb0b..d9e2145d92 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -33,17 +33,17 @@ from swift.common.exceptions import AuditException class ObjectAuditor(object): """Audit objects.""" - def __init__(self, server_conf, auditor_conf): - self.logger = get_logger(auditor_conf, 'object-auditor') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - self.interval = int(auditor_conf.get('interval', 1800)) - swift_dir = server_conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 1800)) + swift_dir = conf.get('swift_dir', '/etc/swift') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.node_timeout = int(auditor_conf.get('node_timeout', 10)) - self.conn_timeout = float(auditor_conf.get('conn_timeout', 0.5)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.passes = 0 self.quarantines = 0 self.errors = 0 diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index de3bea3972..8848748a94 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -215,7 +215,7 @@ class ObjectReplicator(object): 'vm_test_mode', 'no').lower() in ('yes', 'true', 'on', '1') self.swift_dir = conf.get('swift_dir', '/etc/swift') self.port = int(conf.get('bind_port', 6000)) - self.concurrency = int(conf.get('replication_concurrency', 1)) + self.concurrency = int(conf.get('concurrency', 1)) self.timeout = conf['timeout'] self.stats_interval = int(conf['stats_interval']) self.object_ring = Ring(join(self.swift_dir, 'object.ring.gz')) diff --git a/swift/obj/server.py b/swift/obj/server.py index 39c2ca2276..c76f44f331 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -36,9 +36,8 @@ from xattr import getxattr, setxattr from eventlet import sleep, Timeout from swift.common.utils import mkdirs, normalize_timestamp, \ - storage_directory, hash_path, get_logger, renamer, fallocate, \ - split_path, drop_buffer_cache -from swift.common.healthcheck import healthcheck + storage_directory, hash_path, renamer, fallocate, \ + split_path, drop_buffer_cache, get_logger from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_xml_encodable @@ -242,8 +241,6 @@ class DiskFile(object): class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" - log_name = 'object' - def __init__(self, conf): """ Creates a new WSGI application for the Swift Object Server. An @@ -251,7 +248,7 @@ class ObjectController(object): /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ - self.logger = get_logger(conf, self.log_name) + self.logger = get_logger(conf) self.devices = conf.get('devices', '/srv/node/') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') @@ -560,9 +557,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - if req.path_info == '/healthcheck': - return healthcheck(req)(env, start_response) - elif not check_xml_encodable(req.path_info): + if not check_xml_encodable(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: @@ -596,3 +591,9 @@ class ObjectController(object): if slow > 0: sleep(slow) return res(env, start_response) + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI object server apps""" + conf = global_conf.copy() + conf.update(local_conf) + return ObjectController(conf) diff --git a/swift/obj/updater.py b/swift/obj/updater.py index e974364090..7fa9182823 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -32,19 +32,19 @@ from swift.obj.server import ASYNCDIR class ObjectUpdater(object): """Update object information in container listings.""" - def __init__(self, server_conf, updater_conf): - self.logger = get_logger(updater_conf, 'object-updater') - self.devices = server_conf.get('devices', '/srv/node') - self.mount_check = server_conf.get('mount_check', 'true').lower() in \ + def __init__(self, conf): + self.logger = get_logger(conf) + self.devices = conf.get('devices', '/srv/node') + self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') - swift_dir = server_conf.get('swift_dir', '/etc/swift') - self.interval = int(updater_conf.get('interval', 300)) + swift_dir = conf.get('swift_dir', '/etc/swift') + self.interval = int(conf.get('interval', 300)) self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.container_ring = None - self.concurrency = int(updater_conf.get('concurrency', 1)) - self.slowdown = float(updater_conf.get('slowdown', 0.01)) - self.node_timeout = int(updater_conf.get('node_timeout', 10)) - self.conn_timeout = float(updater_conf.get('conn_timeout', 0.5)) + self.concurrency = int(conf.get('concurrency', 1)) + self.slowdown = float(conf.get('slowdown', 0.01)) + self.node_timeout = int(conf.get('node_timeout', 10)) + self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.successes = 0 self.failures = 0 diff --git a/swift/proxy/server.py b/swift/proxy/server.py index b1efd6ea6f..9ba79e13a8 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -31,9 +31,9 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \ from webob import Request, Response from swift.common.ring import Ring -from swift.common.utils import get_logger, normalize_timestamp, split_path +from swift.common.utils import get_logger, normalize_timestamp, split_path, \ + cache_from_env from swift.common.bufferedhttp import http_connect -from swift.common.healthcheck import HealthCheckController from swift.common.constraints import check_object_creation, check_metadata, \ MAX_FILE_SIZE, check_xml_encodable from swift.common.exceptions import ChunkReadTimeout, \ @@ -47,7 +47,7 @@ def update_headers(response, headers): Helper function to update headers in the response. :param response: webob.Response object - :param headers: dictionary headers + :parm headers: dictionary headers """ if hasattr(headers, 'items'): headers = headers.items() @@ -930,14 +930,12 @@ class AccountController(Controller): class BaseApplication(object): """Base WSGI application for the proxy server""" - log_name = 'base_application' - - def __init__(self, conf, memcache, logger=None, account_ring=None, + def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): - if logger: - self.logger = logger + if logger is None: + self.logger = get_logger(conf) else: - self.logger = get_logger(conf, self.log_name) + self.logger = logger if conf is None: conf = {} swift_dir = conf.get('swift_dir', '/etc/swift') @@ -991,8 +989,6 @@ class BaseApplication(object): return ContainerController, d elif account and not container and not obj: return AccountController, d - elif version and version == 'healthcheck': - return HealthCheckController, d return None, d def __call__(self, env, start_response): @@ -1004,6 +1000,8 @@ class BaseApplication(object): :param start_response: WSGI callable """ try: + if self.memcache is None: + self.memcache = cache_from_env(env) req = self.update_request(Request(env)) if 'eventlet.posthooks' in env: env['eventlet.posthooks'].append( @@ -1048,13 +1046,6 @@ class BaseApplication(object): controller, path_parts = self.get_controller(req.path) except ValueError: return HTTPNotFound(request=req) - if controller == HealthCheckController: - controller = controller(self, **path_parts) - controller.trans_id = req.headers.get('x-cf-trans-id', '-') - if req.method == 'GET': - return controller.GET(req) - return HTTPMethodNotAllowed(request=req) - if not check_xml_encodable(req.path_info): return HTTPPreconditionFailed(request=req, body='Invalid UTF8') if not controller: @@ -1086,8 +1077,6 @@ class BaseApplication(object): class Application(BaseApplication): """WSGI application for the proxy server.""" - log_name = 'proxy' - def handle_request(self, req): """ Wraps the BaseApplication.handle_request and logs the request. @@ -1181,3 +1170,7 @@ class Application(BaseApplication): return Response(status='498 Rate Limited', body='Slow down', request=req) return None + +def app_factory(global_conf, **local_conf): + """paste.deploy app factory for creating WSGI proxy apps.""" + return Application(local_conf) diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 84ef0ed897..c4f445345d 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -795,33 +795,6 @@ class TestAccountController(unittest.TestCase): listing.append(node2.firstChild.nodeValue) self.assertEquals(listing, ['sub.1.0', 'sub.1.1', 'sub.1.2']) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_through_call(self): inbuf = StringIO() errbuf = StringIO() diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 9bb23a925d..31495401f2 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -75,7 +75,7 @@ class TestAuthServer(unittest.TestCase): 'auth_server') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) - self.conf = {'swift_dir': self.testdir} + self.conf = {'swift_dir': self.testdir, 'log_name': 'auth'} self.controller = auth_server.AuthController(self.conf, FakeRing()) def tearDown(self): diff --git a/test/unit/common/middleware/__init__.py b/test/unit/common/middleware/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/common/test_auth.py b/test/unit/common/middleware/test_auth.py similarity index 99% rename from test/unit/common/test_auth.py rename to test/unit/common/middleware/test_auth.py index cf35827522..cd5a02c91d 100644 --- a/test/unit/common/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -23,7 +23,7 @@ from contextlib import contextmanager import eventlet from webob import Request -from swift.common import auth +from swift.common.middleware import auth # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) diff --git a/test/unit/common/middleware/test_cache.py b/test/unit/common/middleware/test_cache.py new file mode 100644 index 0000000000..a83233e2eb --- /dev/null +++ b/test/unit/common/middleware/test_cache.py @@ -0,0 +1,42 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from webob import Request + +from swift.common.middleware import cache +from swift.common.memcached import MemcacheRing + +class FakeApp(object): + def __call__(self, env, start_response): + return env + +def start_response(*args): + pass + +class TestCacheMiddleware(unittest.TestCase): + + def setUp(self): + self.app = cache.CacheMiddleware(FakeApp(), {}) + + def test_cache_middleware(self): + req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertTrue('swift.cache' in resp) + self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing)) + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_healthcheck.py b/test/unit/common/middleware/test_healthcheck.py similarity index 54% rename from test/unit/common/test_healthcheck.py rename to test/unit/common/middleware/test_healthcheck.py index 3256d474c0..fc1a2a198a 100644 --- a/test/unit/common/test_healthcheck.py +++ b/test/unit/common/middleware/test_healthcheck.py @@ -17,17 +17,29 @@ import unittest from webob import Request -from swift.common import healthcheck +from swift.common.middleware import healthcheck +class FakeApp(object): + def __call__(self, env, start_response): + return "FAKE APP" + +def start_response(*args): + pass class TestHealthCheck(unittest.TestCase): - def test_healthcheck(self): - controller = healthcheck.HealthCheckController() - req = Request.blank('/any/path', environ={'REQUEST_METHOD': 'GET'}) - resp = controller.GET(req) - self.assertEquals(resp.status_int, 200) + def setUp(self): + self.app = healthcheck.HealthCheckMiddleware(FakeApp()) + def test_healthcheck(self): + req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, ['OK']) + + def test_healtcheck_pass(self): + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) + resp = self.app(req.environ, start_response) + self.assertEquals(resp, 'FAKE APP') if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index bd247772f0..981deab184 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -139,7 +139,7 @@ class TestDBReplicator(unittest.TestCase): self.assertEquals(conn.post(1, 2, 3), None) def test_rsync_file(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) with _mock_process(-1): fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} self.assertEquals(False, @@ -150,13 +150,13 @@ class TestDBReplicator(unittest.TestCase): replicator._rsync_file('/some/file', 'remote:/some/file')) def test_rsync_db(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._rsync_file = lambda *args: True fake_device = {'ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(FakeBroker(), fake_device, PostReplHttp(), 'abcd') def test_in_sync(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) self.assertEquals(replicator._in_sync( {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, @@ -171,16 +171,16 @@ class TestDBReplicator(unittest.TestCase): FakeBroker(), -1)), False) def test_replicate_once(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator.replicate_once() def test_usync(self): fake_http = PostReplHttp() - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890') def test_repl_to_node(self): - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000} fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b', 'created_at': 100, 'put_timestamp': 0, @@ -192,13 +192,13 @@ class TestDBReplicator(unittest.TestCase): def test_stats(self): # I'm not sure how to test that this logs the right thing, # but we can at least make sure it gets covered. - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._zero_stats() replicator._report_stats() def test_replicate_object(self): db_replicator.lock_parent_directory = lock_parent_directory - replicator = TestReplicator({}, {}) + replicator = TestReplicator({}) replicator._replicate_object('0', 'file', 'node_id') diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index b38c5489e4..08b15e4203 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -551,33 +551,6 @@ class TestContainerController(unittest.TestCase): {"name":"US/TX","hash":"x","bytes":0,"content_type":"text/plain", "last_modified":"1970-01-01T00:00:01"}]) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_through_call(self): inbuf = StringIO() errbuf = StringIO() diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index a4e88eba43..108334363c 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -61,10 +61,14 @@ class TestContainerUpdater(unittest.TestCase): rmtree(self.testdir, ignore_errors=1) def test_creation(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '2', + 'node_timeout': '5', + }) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) @@ -74,10 +78,14 @@ class TestContainerUpdater(unittest.TestCase): self.assert_(cu.get_account_ring() is not None) def test_update_once_single_threaded(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) cu.update_once_single_threaded() containers_dir = os.path.join(self.sda1, container_server.DATADIR) os.mkdir(containers_dir) @@ -152,10 +160,14 @@ class TestContainerUpdater(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 3) def test_unicode(self): - cu = container_updater.ContainerUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = container_updater.ContainerUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) containers_dir = os.path.join(self.sda1, container_server.DATADIR) os.mkdir(containers_dir) subdir = os.path.join(containers_dir, 'subdir') diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 316f5763e4..17308c614a 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -768,33 +768,6 @@ class TestObjectController(unittest.TestCase): timestamp + '.ts') self.assert_(os.path.isfile(objfile)) - def test_healthcheck(self): - inbuf = StringIO() - errbuf = StringIO() - outbuf = StringIO() - - def start_response(*args): - """ Sends args to outbuf """ - outbuf.writelines(args) - - self.object_controller.__call__({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/healthcheck', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': inbuf, - 'wsgi.errors': errbuf, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False}, - start_response) - self.assertEquals(errbuf.getvalue(), '') - self.assertEquals(outbuf.getvalue()[:4], '200 ') - def test_call(self): """ Test swift.object_server.ObjectController.__call__ """ inbuf = StringIO() diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index e5de8df4fd..6c7aa70c6c 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -53,10 +53,14 @@ class TestObjectUpdater(unittest.TestCase): rmtree(self.testdir, ignore_errors=1) def test_creation(self): - cu = object_updater.ObjectUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) + cu = object_updater.ObjectUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '2', + 'node_timeout': '5', + }) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) @@ -66,10 +70,14 @@ class TestObjectUpdater(unittest.TestCase): self.assert_(cu.get_container_ring() is not None) def test_update_once_single_threaded(self): - cu = object_updater.ObjectUpdater( - {'devices': self.devices_dir, 'mount_check': 'false', - 'swift_dir': self.testdir}, - {'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) + cu = object_updater.ObjectUpdater({ + 'devices': self.devices_dir, + 'mount_check': 'false', + 'swift_dir': self.testdir, + 'interval': '1', + 'concurrency': '1', + 'node_timeout': '15', + }) cu.update_once_single_threaded() async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) os.mkdir(async_dir) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index eb3b4254da..2aad3a26fc 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1100,17 +1100,6 @@ class TestObjectController(unittest.TestCase): obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) try: - # healthcheck test - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('GET /healthcheck HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 200' - self.assertEquals(headers[:len(exp)], exp) - body = fd.read() - self.assertEquals(body, 'OK') # Check bad version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() @@ -1129,15 +1118,6 @@ class TestObjectController(unittest.TestCase): headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) - # Check bad method - sock = connect_tcp(('localhost', prolis.getsockname()[1])) - fd = sock.makefile() - fd.write('LICK /healthcheck HTTP/1.1\r\nHost: localhost\r\n' - 'Connection: close\r\nContent-Length: 0\r\n\r\n') - fd.flush() - headers = readuntil2crlfs(fd) - exp = 'HTTP/1.1 405' - self.assertEquals(headers[:len(exp)], exp) # Check blacklist prosrv.rate_limit_blacklist = ['a'] sock = connect_tcp(('localhost', prolis.getsockname()[1]))