I rewrote swift-init cause I wanted it to do somethings it didn't, and I
tried to make some of the error conditions a little better too. It has some new features, but overall it's behavior is mostly compatible with the old swift-init (input, return codes) - but if someone had some weird greps on the output they're hozed. swift-init --help is a good place to start some highlights might be: $swift-init start main --wait # wait for all "main" services to start, printing any errors to console $swift-init rest wait # same as above but for replicators, updaters & auditors $swift-init stop object -c1 # kill the first object-server "node" $swift-init object-server status # make sure it's dead - yay status! $swift-init proxy auth reload # devauth? srsly? $swift-init *-replicator once -n # run the object, container, and account replicators in "once mode" and watch them log to console until they're finished ... probably some other stuff too. Looking forward to feedback...
This commit is contained in:
Clay Gerrard
Tarmac
commit
72ca13e0c0
212
bin/swift-init
Executable file → Normal file
212
bin/swift-init
Executable file → Normal file
@ -14,180 +14,60 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import with_statement
|
||||
import errno
|
||||
import glob
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from optparse import OptionParser
|
||||
|
||||
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
|
||||
'container-replicator', 'container-server', 'container-updater',
|
||||
'object-auditor', 'object-server', 'object-replicator', 'object-updater',
|
||||
'proxy-server', 'account-replicator', 'auth-server', 'account-reaper']
|
||||
GRACEFUL_SHUTDOWN_SERVERS = ['account-server', 'container-server',
|
||||
'object-server', 'proxy-server', 'auth-server']
|
||||
MAX_DESCRIPTORS = 32768
|
||||
MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB
|
||||
from swift.common.manager import Server, Manager, UnknownCommandError
|
||||
|
||||
_junk, server, command = sys.argv
|
||||
if server == 'all':
|
||||
servers = ALL_SERVERS
|
||||
else:
|
||||
if '-' not in server:
|
||||
server = '%s-server' % server
|
||||
servers = [server]
|
||||
command = command.lower()
|
||||
USAGE = """%prog <server> [<server> ...] <command> [options]
|
||||
|
||||
def pid_files(server):
|
||||
if os.path.exists('/var/run/swift/%s.pid' % server):
|
||||
pid_files = ['/var/run/swift/%s.pid' % server]
|
||||
else:
|
||||
pid_files = glob.glob('/var/run/swift/%s/*.pid' % server)
|
||||
for pid_file in pid_files:
|
||||
pid = int(open(pid_file).read().strip())
|
||||
yield pid_file, pid
|
||||
Commands:
|
||||
""" + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])
|
||||
|
||||
def do_start(server, once=False):
|
||||
server_type = '-'.join(server.split('-')[:-1])
|
||||
|
||||
for pid_file, pid in pid_files(server):
|
||||
if os.path.exists('/proc/%s' % pid):
|
||||
print "%s appears to already be running: %s" % (server, pid_file)
|
||||
return
|
||||
else:
|
||||
print "Removing stale pid file %s" % pid_file
|
||||
os.unlink(pid_file)
|
||||
def main():
|
||||
parser = OptionParser(USAGE)
|
||||
parser.add_option('-v', '--verbose', action="store_true",
|
||||
default=False, help="display verbose output")
|
||||
parser.add_option('-w', '--no-wait', action="store_false", dest="wait",
|
||||
default=True, help="won't wait for server to start "
|
||||
"before returning")
|
||||
parser.add_option('-o', '--once', action="store_true",
|
||||
default=False, help="only run one pass of daemon")
|
||||
# this is a negative option, default is options.daemon = True
|
||||
parser.add_option('-n', '--no-daemon', action="store_false", dest="daemon",
|
||||
default=True, help="start server interactively")
|
||||
parser.add_option('-g', '--graceful', action="store_true",
|
||||
default=False, help="send SIGHUP to supporting servers")
|
||||
parser.add_option('-c', '--config-num', metavar="N", type="int",
|
||||
dest="number", default=0,
|
||||
help="send command to the Nth server only")
|
||||
options, args = parser.parse_args()
|
||||
|
||||
if len(args) < 2:
|
||||
parser.print_help()
|
||||
print 'ERROR: specify server(s) and command'
|
||||
return 1
|
||||
|
||||
command = args[-1]
|
||||
servers = args[:-1]
|
||||
|
||||
# this is just a silly swap for me cause I always try to "start main"
|
||||
commands = dict(Manager.list_commands()).keys()
|
||||
if command not in commands and servers[0] in commands:
|
||||
servers.append(command)
|
||||
command = servers.pop(0)
|
||||
|
||||
manager = Manager(servers)
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE,
|
||||
(MAX_DESCRIPTORS, MAX_DESCRIPTORS))
|
||||
resource.setrlimit(resource.RLIMIT_DATA,
|
||||
(MAX_MEMORY, MAX_MEMORY))
|
||||
except ValueError:
|
||||
print "Unable to increase file descriptor limit. Running as non-root?"
|
||||
os.environ['PYTHON_EGG_CACHE'] = '/tmp'
|
||||
status = manager.run_command(command, **options.__dict__)
|
||||
except UnknownCommandError:
|
||||
parser.print_help()
|
||||
print 'ERROR: unknown command, %s' % command
|
||||
status = 1
|
||||
|
||||
def write_pid_file(pid_file, pid):
|
||||
dir, file = os.path.split(pid_file)
|
||||
if not os.path.exists(dir):
|
||||
try:
|
||||
os.makedirs(dir)
|
||||
except OSError, err:
|
||||
if err.errno == errno.EACCES:
|
||||
sys.exit('Unable to create %s. Running as non-root?' % dir)
|
||||
fp = open(pid_file, 'w')
|
||||
fp.write('%d\n' % pid)
|
||||
fp.close()
|
||||
return 1 if status else 0
|
||||
|
||||
def launch(ini_file, pid_file):
|
||||
cmd = 'swift-%s' % server
|
||||
args = [server, ini_file]
|
||||
if once:
|
||||
print 'Running %s once' % server
|
||||
args.append('once')
|
||||
else:
|
||||
print 'Starting %s' % server
|
||||
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
os.setsid()
|
||||
with open(os.devnull, 'r+b') as nullfile:
|
||||
for desc in (0, 1, 2): # close stdio
|
||||
try:
|
||||
os.dup2(nullfile.fileno(), desc)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
if once:
|
||||
os.execlp('swift-%s' % server, server,
|
||||
ini_file, 'once')
|
||||
else:
|
||||
os.execlp('swift-%s' % server, server, ini_file)
|
||||
except OSError:
|
||||
print 'unable to launch %s' % server
|
||||
sys.exit(0)
|
||||
else:
|
||||
write_pid_file(pid_file, pid)
|
||||
|
||||
ini_file = '/etc/swift/%s-server.conf' % server_type
|
||||
if os.path.exists(ini_file):
|
||||
# single config file over-rides config dirs
|
||||
pid_file = '/var/run/swift/%s.pid' % server
|
||||
launch_args = [(ini_file, pid_file)]
|
||||
elif os.path.exists('/etc/swift/%s-server/' % server_type):
|
||||
# found config directory, searching for config file(s)
|
||||
launch_args = []
|
||||
for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' \
|
||||
% server_type)):
|
||||
pid_file = '/var/run/swift/%s/%d.pid' % (server, num)
|
||||
# start a server for each ini_file found
|
||||
launch_args.append((ini_file, pid_file))
|
||||
else:
|
||||
# maybe there's a config file(s) out there, but I couldn't find it!
|
||||
print 'Unable to locate config file for %s. %s does not exist?' % \
|
||||
(server, ini_file)
|
||||
return
|
||||
|
||||
# start all servers
|
||||
for ini_file, pid_file in launch_args:
|
||||
launch(ini_file, pid_file)
|
||||
|
||||
def do_stop(server, graceful=False):
|
||||
if graceful and server in GRACEFUL_SHUTDOWN_SERVERS:
|
||||
sig = signal.SIGHUP
|
||||
else:
|
||||
sig = signal.SIGTERM
|
||||
|
||||
did_anything = False
|
||||
pfiles = pid_files(server)
|
||||
for pid_file, pid in pfiles:
|
||||
did_anything = True
|
||||
try:
|
||||
print 'Stopping %s pid: %s signal: %s' % (server, pid, sig)
|
||||
os.kill(pid, sig)
|
||||
except OSError:
|
||||
print "Process %d not running" % pid
|
||||
try:
|
||||
os.unlink(pid_file)
|
||||
except OSError:
|
||||
pass
|
||||
for pid_file, pid in pfiles:
|
||||
for _junk in xrange(150): # 15 seconds
|
||||
if not os.path.exists('/proc/%s' % pid):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
print 'Waited 15 seconds for pid %s (%s) to die; giving up' % \
|
||||
(pid, pid_file)
|
||||
if not did_anything:
|
||||
print 'No %s running' % server
|
||||
|
||||
if command == 'start':
|
||||
for server in servers:
|
||||
do_start(server)
|
||||
|
||||
if command == 'stop':
|
||||
for server in servers:
|
||||
do_stop(server)
|
||||
|
||||
if command == 'shutdown':
|
||||
for server in servers:
|
||||
do_stop(server, graceful=True)
|
||||
|
||||
if command == 'restart':
|
||||
for server in servers:
|
||||
do_stop(server)
|
||||
for server in servers:
|
||||
do_start(server)
|
||||
|
||||
if command == 'reload' or command == 'force-reload':
|
||||
for server in servers:
|
||||
do_stop(server, graceful=True)
|
||||
do_start(server)
|
||||
|
||||
if command == 'once':
|
||||
for server in servers:
|
||||
do_start(server, once=True)
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@ -531,7 +531,6 @@ Setting up scripts for running Swift
|
||||
#!/bin/bash
|
||||
|
||||
swift-init all stop
|
||||
sleep 5
|
||||
sudo umount /mnt/sdb1
|
||||
sudo mkfs.xfs -f -i size=1024 /dev/sdb1
|
||||
sudo mount /mnt/sdb1
|
||||
@ -573,12 +572,9 @@ Setting up scripts for running Swift
|
||||
|
||||
#!/bin/bash
|
||||
|
||||
swift-init main start
|
||||
# The auth-server line is only needed for DevAuth:
|
||||
swift-init auth-server start
|
||||
swift-init proxy-server start
|
||||
swift-init account-server start
|
||||
swift-init container-server start
|
||||
swift-init object-server start
|
||||
|
||||
#. For Swauth (not needed for DevAuth), create `~/bin/recreateaccounts`::
|
||||
|
||||
@ -600,15 +596,7 @@ Setting up scripts for running Swift
|
||||
# /etc/swift/auth-server.conf). This swift-auth-recreate-accounts line
|
||||
# is only needed for DevAuth:
|
||||
swift-auth-recreate-accounts -K devauth
|
||||
swift-init object-updater start
|
||||
swift-init container-updater start
|
||||
swift-init object-replicator start
|
||||
swift-init container-replicator start
|
||||
swift-init account-replicator start
|
||||
swift-init object-auditor start
|
||||
swift-init container-auditor start
|
||||
swift-init account-auditor start
|
||||
swift-init account-reaper start
|
||||
swift-init rest start
|
||||
|
||||
#. `chmod +x ~/bin/*`
|
||||
#. `remakerings`
|
||||
|
||||
@ -116,6 +116,13 @@ MemCacheD
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Manager
|
||||
=========
|
||||
|
||||
.. automodule:: swift.common.manager
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Ratelimit
|
||||
=========
|
||||
|
||||
|
||||
@ -39,8 +39,8 @@ class Daemon(object):
|
||||
def run(self, once=False, **kwargs):
|
||||
"""Run the daemon"""
|
||||
utils.validate_configuration()
|
||||
utils.capture_stdio(self.logger, **kwargs)
|
||||
utils.drop_privileges(self.conf.get('user', 'swift'))
|
||||
utils.capture_stdio(self.logger, **kwargs)
|
||||
|
||||
def kill_children(*args):
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
|
||||
605
swift/common/manager.py
Normal file
605
swift/common/manager.py
Normal file
@ -0,0 +1,605 @@
|
||||
# Copyright (c) 2010-2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import with_statement
|
||||
import functools
|
||||
import errno
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
from swift.common.utils import search_tree, remove_file, write_file
|
||||
|
||||
SWIFT_DIR = '/etc/swift'
|
||||
RUN_DIR = '/var/run/swift'
|
||||
|
||||
# auth-server has been removed from ALL_SERVERS, start it explicitly
|
||||
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
|
||||
'container-replicator', 'container-server', 'container-updater',
|
||||
'object-auditor', 'object-server', 'object-replicator', 'object-updater',
|
||||
'proxy-server', 'account-replicator', 'account-reaper']
|
||||
MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server',
|
||||
'object-server']
|
||||
REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS]
|
||||
GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server']
|
||||
START_ONCE_SERVERS = REST_SERVERS
|
||||
|
||||
KILL_WAIT = 15 # seconds to wait for servers to die
|
||||
|
||||
MAX_DESCRIPTORS = 32768
|
||||
MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB
|
||||
|
||||
|
||||
def setup_env():
|
||||
"""Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp
|
||||
"""
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE,
|
||||
(MAX_DESCRIPTORS, MAX_DESCRIPTORS))
|
||||
resource.setrlimit(resource.RLIMIT_DATA,
|
||||
(MAX_MEMORY, MAX_MEMORY))
|
||||
except ValueError:
|
||||
print _("WARNING: Unable to increase file descriptor limit. "
|
||||
"Running as non-root?")
|
||||
|
||||
os.environ['PYTHON_EGG_CACHE'] = '/tmp'
|
||||
|
||||
|
||||
def command(func):
|
||||
"""
|
||||
Decorator to declare which methods are accessible as commands, commands
|
||||
always return 1 or 0, where 0 should indicate success.
|
||||
|
||||
:param func: function to make public
|
||||
"""
|
||||
func.publicly_accessible = True
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapped(*a, **kw):
|
||||
rv = func(*a, **kw)
|
||||
return 1 if rv else 0
|
||||
return wrapped
|
||||
|
||||
|
||||
def watch_server_pids(server_pids, interval=1, **kwargs):
|
||||
"""Monitor a collection of server pids yeilding back those pids that
|
||||
aren't responding to signals.
|
||||
|
||||
:param server_pids: a dict, lists of pids [int,...] keyed on
|
||||
Server objects
|
||||
"""
|
||||
status = {}
|
||||
start = time.time()
|
||||
end = start + interval
|
||||
server_pids = dict(server_pids) # make a copy
|
||||
while True:
|
||||
for server, pids in server_pids.items():
|
||||
for pid in pids:
|
||||
try:
|
||||
# let pid stop if it wants to
|
||||
os.waitpid(pid, os.WNOHANG)
|
||||
except OSError, e:
|
||||
if e.errno not in (errno.ECHILD, errno.ESRCH):
|
||||
raise # else no such child/process
|
||||
# check running pids for server
|
||||
status[server] = server.get_running_pids(**kwargs)
|
||||
for pid in pids:
|
||||
# original pids no longer in running pids!
|
||||
if pid not in status[server]:
|
||||
yield server, pid
|
||||
# update active pids list using running_pids
|
||||
server_pids[server] = status[server]
|
||||
if not [p for server, pids in status.items() for p in pids]:
|
||||
# no more running pids
|
||||
break
|
||||
if time.time() > end:
|
||||
break
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
class UnknownCommandError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Manager():
|
||||
"""Main class for performing commands on groups of servers.
|
||||
|
||||
:param servers: list of server names as strings
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, servers):
|
||||
server_names = set()
|
||||
for server in servers:
|
||||
if server == 'all':
|
||||
server_names.update(ALL_SERVERS)
|
||||
elif server == 'main':
|
||||
server_names.update(MAIN_SERVERS)
|
||||
elif server == 'rest':
|
||||
server_names.update(REST_SERVERS)
|
||||
elif '*' in server:
|
||||
# convert glob to regex
|
||||
server_names.update([s for s in ALL_SERVERS if
|
||||
re.match(server.replace('*', '.*'), s)])
|
||||
else:
|
||||
server_names.add(server)
|
||||
|
||||
self.servers = set()
|
||||
for name in server_names:
|
||||
self.servers.add(Server(name))
|
||||
|
||||
@command
|
||||
def status(self, **kwargs):
|
||||
"""display status of tracked pids for server
|
||||
"""
|
||||
status = 0
|
||||
for server in self.servers:
|
||||
status += server.status(**kwargs)
|
||||
return status
|
||||
|
||||
@command
|
||||
def start(self, **kwargs):
|
||||
"""starts a server
|
||||
"""
|
||||
setup_env()
|
||||
status = 0
|
||||
|
||||
for server in self.servers:
|
||||
server.launch(**kwargs)
|
||||
if not kwargs.get('daemon', True):
|
||||
for server in self.servers:
|
||||
try:
|
||||
status += server.interact(**kwargs)
|
||||
except KeyboardInterrupt:
|
||||
print _('\nuser quit')
|
||||
self.stop(**kwargs)
|
||||
break
|
||||
elif kwargs.get('wait', True):
|
||||
for server in self.servers:
|
||||
status += server.wait(**kwargs)
|
||||
return status
|
||||
|
||||
@command
|
||||
def no_wait(self, **kwargs):
|
||||
"""spawn server and return immediately
|
||||
"""
|
||||
kwargs['wait'] = False
|
||||
return self.start(**kwargs)
|
||||
|
||||
@command
|
||||
def no_daemon(self, **kwargs):
|
||||
"""start a server interactively
|
||||
"""
|
||||
kwargs['daemon'] = False
|
||||
return self.start(**kwargs)
|
||||
|
||||
@command
|
||||
def once(self, **kwargs):
|
||||
"""start server and run one pass on supporting daemons
|
||||
"""
|
||||
kwargs['once'] = True
|
||||
return self.start(**kwargs)
|
||||
|
||||
@command
|
||||
def stop(self, **kwargs):
|
||||
"""stops a server
|
||||
"""
|
||||
server_pids = {}
|
||||
for server in self.servers:
|
||||
signaled_pids = server.stop(**kwargs)
|
||||
if not signaled_pids:
|
||||
print _('No %s running') % server
|
||||
else:
|
||||
server_pids[server] = signaled_pids
|
||||
|
||||
# all signaled_pids, i.e. list(itertools.chain(*server_pids.values()))
|
||||
signaled_pids = [p for server, pids in server_pids.items()
|
||||
for p in pids]
|
||||
# keep track of the pids yeiled back as killed for all servers
|
||||
killed_pids = set()
|
||||
for server, killed_pid in watch_server_pids(server_pids,
|
||||
interval=KILL_WAIT, **kwargs):
|
||||
print _("%s (%s) appears to have stopped") % (server, killed_pid)
|
||||
killed_pids.add(killed_pid)
|
||||
if not killed_pids.symmetric_difference(signaled_pids):
|
||||
# all proccesses have been stopped
|
||||
return 0
|
||||
|
||||
# reached interval n watch_pids w/o killing all servers
|
||||
for server, pids in server_pids.items():
|
||||
if not killed_pids.issuperset(pids):
|
||||
# some pids of this server were not killed
|
||||
print _('Waited %s seconds for %s to die; giving up') % (
|
||||
KILL_WAIT, server)
|
||||
return 1
|
||||
|
||||
@command
|
||||
def shutdown(self, **kwargs):
|
||||
"""allow current requests to finish on supporting servers
|
||||
"""
|
||||
kwargs['graceful'] = True
|
||||
status = 0
|
||||
status += self.stop(**kwargs)
|
||||
return status
|
||||
|
||||
@command
|
||||
def restart(self, **kwargs):
|
||||
"""stops then restarts server
|
||||
"""
|
||||
status = 0
|
||||
status += self.stop(**kwargs)
|
||||
status += self.start(**kwargs)
|
||||
return status
|
||||
|
||||
@command
|
||||
def reload(self, **kwargs):
|
||||
"""graceful shutdown then restart on supporting servers
|
||||
"""
|
||||
kwargs['graceful'] = True
|
||||
status = 0
|
||||
for server in self.servers:
|
||||
m = Manager([server.server])
|
||||
status += m.stop(**kwargs)
|
||||
status += m.start(**kwargs)
|
||||
return status
|
||||
|
||||
@command
|
||||
def force_reload(self, **kwargs):
|
||||
"""alias for reload
|
||||
"""
|
||||
return self.reload(**kwargs)
|
||||
|
||||
def get_command(self, cmd):
|
||||
"""Find and return the decorated method named like cmd
|
||||
|
||||
:param cmd: the command to get, a string, if not found raises
|
||||
UnknownCommandError
|
||||
|
||||
"""
|
||||
cmd = cmd.lower().replace('-', '_')
|
||||
try:
|
||||
f = getattr(self, cmd)
|
||||
except AttributeError:
|
||||
raise UnknownCommandError(cmd)
|
||||
if not hasattr(f, 'publicly_accessible'):
|
||||
raise UnknownCommandError(cmd)
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
def list_commands(cls):
|
||||
"""Get all publicly accessible commands
|
||||
|
||||
:returns: a list of string tuples (cmd, help), the method names who are
|
||||
decorated as commands
|
||||
"""
|
||||
get_method = lambda cmd: getattr(cls, cmd)
|
||||
return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip())
|
||||
for x in dir(cls) if
|
||||
getattr(get_method(x), 'publicly_accessible', False)])
|
||||
|
||||
def run_command(self, cmd, **kwargs):
|
||||
"""Find the named command and run it
|
||||
|
||||
:param cmd: the command name to run
|
||||
|
||||
"""
|
||||
f = self.get_command(cmd)
|
||||
return f(**kwargs)
|
||||
|
||||
|
||||
class Server():
|
||||
"""Manage operations on a server or group of servers of similar type
|
||||
|
||||
:param server: name of server
|
||||
"""
|
||||
|
||||
def __init__(self, server):
|
||||
if '-' not in server:
|
||||
server = '%s-server' % server
|
||||
self.server = server.lower()
|
||||
self.type = server.rsplit('-', 1)[0]
|
||||
self.cmd = 'swift-%s' % server
|
||||
self.procs = []
|
||||
|
||||
def __str__(self):
|
||||
return self.server
|
||||
|
||||
def __repr__(self):
|
||||
return "%s(%s)" % (self.__class__.__name__, repr(str(self)))
|
||||
|
||||
def __hash__(self):
|
||||
return hash(str(self))
|
||||
|
||||
def __eq__(self, other):
|
||||
try:
|
||||
return self.server == other.server
|
||||
except AttributeError:
|
||||
return False
|
||||
|
||||
def get_pid_file_name(self, conf_file):
|
||||
"""Translate conf_file to a corresponding pid_file
|
||||
|
||||
:param conf_file: an conf_file for this server, a string
|
||||
|
||||
:returns: the pid_file for this conf_file
|
||||
|
||||
"""
|
||||
return conf_file.replace(
|
||||
os.path.normpath(SWIFT_DIR), RUN_DIR, 1).replace(
|
||||
'%s-server' % self.type, self.server, 1).rsplit(
|
||||
'.conf', 1)[0] + '.pid'
|
||||
|
||||
def get_conf_file_name(self, pid_file):
|
||||
"""Translate pid_file to a corresponding conf_file
|
||||
|
||||
:param pid_file: a pid_file for this server, a string
|
||||
|
||||
:returns: the conf_file for this pid_file
|
||||
|
||||
"""
|
||||
return pid_file.replace(
|
||||
os.path.normpath(RUN_DIR), SWIFT_DIR, 1).replace(
|
||||
self.server, '%s-server' % self.type, 1).rsplit(
|
||||
'.pid', 1)[0] + '.conf'
|
||||
|
||||
def conf_files(self, **kwargs):
|
||||
"""Get conf files for this server
|
||||
|
||||
:param: number, if supplied will only lookup the nth server
|
||||
|
||||
:returns: list of conf files
|
||||
"""
|
||||
found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type,
|
||||
'.conf')
|
||||
number = kwargs.get('number')
|
||||
if number:
|
||||
try:
|
||||
conf_files = [found_conf_files[number - 1]]
|
||||
except IndexError:
|
||||
conf_files = []
|
||||
else:
|
||||
conf_files = found_conf_files
|
||||
if not conf_files:
|
||||
# maybe there's a config file(s) out there, but I couldn't find it!
|
||||
if not kwargs.get('quiet'):
|
||||
print _('Unable to locate config %sfor %s') % (
|
||||
('number %s ' % number if number else ''), self.server)
|
||||
if kwargs.get('verbose') and not kwargs.get('quiet'):
|
||||
if found_conf_files:
|
||||
print _('Found configs:')
|
||||
for i, conf_file in enumerate(found_conf_files):
|
||||
print ' %d) %s' % (i + 1, conf_file)
|
||||
|
||||
return conf_files
|
||||
|
||||
def pid_files(self, **kwargs):
|
||||
"""Get pid files for this server
|
||||
|
||||
:param: number, if supplied will only lookup the nth server
|
||||
|
||||
:returns: list of pid files
|
||||
"""
|
||||
pid_files = search_tree(RUN_DIR, '%s*' % self.server, '.pid')
|
||||
if kwargs.get('number', 0):
|
||||
conf_files = self.conf_files(**kwargs)
|
||||
# filter pid_files to match the index of numbered conf_file
|
||||
pid_files = [pid_file for pid_file in pid_files if
|
||||
self.get_conf_file_name(pid_file) in conf_files]
|
||||
return pid_files
|
||||
|
||||
def iter_pid_files(self, **kwargs):
|
||||
"""Generator, yields (pid_file, pids)
|
||||
"""
|
||||
for pid_file in self.pid_files(**kwargs):
|
||||
yield pid_file, int(open(pid_file).read().strip())
|
||||
|
||||
def signal_pids(self, sig, **kwargs):
|
||||
"""Send a signal to pids for this server
|
||||
|
||||
:param sig: signal to send
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
pids = {}
|
||||
for pid_file, pid in self.iter_pid_files(**kwargs):
|
||||
try:
|
||||
if sig != signal.SIG_DFL:
|
||||
print _('Signal %s pid: %s signal: %s') % (self.server,
|
||||
pid, sig)
|
||||
os.kill(pid, sig)
|
||||
except OSError, e:
|
||||
if e.errno == errno.ESRCH:
|
||||
# pid does not exist
|
||||
if kwargs.get('verbose'):
|
||||
print _("Removing stale pid file %s") % pid_file
|
||||
remove_file(pid_file)
|
||||
else:
|
||||
# process exists
|
||||
pids[pid] = pid_file
|
||||
return pids
|
||||
|
||||
def get_running_pids(self, **kwargs):
|
||||
"""Get running pids
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop
|
||||
|
||||
def kill_running_pids(self, **kwargs):
|
||||
"""Kill running pids
|
||||
|
||||
:param graceful: if True, attempt SIGHUP on supporting servers
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
graceful = kwargs.get('graceful')
|
||||
if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS:
|
||||
sig = signal.SIGHUP
|
||||
else:
|
||||
sig = signal.SIGTERM
|
||||
return self.signal_pids(sig, **kwargs)
|
||||
|
||||
def status(self, pids=None, **kwargs):
|
||||
"""Display status of server
|
||||
|
||||
:param: pids, if not supplied pids will be populated automatically
|
||||
:param: number, if supplied will only lookup the nth server
|
||||
|
||||
:returns: 1 if server is not running, 0 otherwise
|
||||
"""
|
||||
if pids is None:
|
||||
pids = self.get_running_pids(**kwargs)
|
||||
if not pids:
|
||||
number = kwargs.get('number', 0)
|
||||
if number:
|
||||
kwargs['quiet'] = True
|
||||
conf_files = self.conf_files(**kwargs)
|
||||
if conf_files:
|
||||
print _("%s #%d not running (%s)") % (self.server, number,
|
||||
conf_files[0])
|
||||
else:
|
||||
print _("No %s running") % self.server
|
||||
return 1
|
||||
for pid, pid_file in pids.items():
|
||||
conf_file = self.get_conf_file_name(pid_file)
|
||||
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
||||
return 0
|
||||
|
||||
def spawn(self, conf_file, once=False, wait=True, daemon=True, **kwargs):
|
||||
"""Launch a subprocess for this server.
|
||||
|
||||
:param conf_file: path to conf_file to use as first arg
|
||||
:param once: boolean, add once argument to command
|
||||
:param wait: boolean, if true capture stdout with a pipe
|
||||
:param daemon: boolean, if true ask server to log to console
|
||||
|
||||
:returns : the pid of the spawned process
|
||||
"""
|
||||
args = [self.cmd, conf_file]
|
||||
if once:
|
||||
args.append('once')
|
||||
if not daemon:
|
||||
# ask the server to log to console
|
||||
args.append('verbose')
|
||||
|
||||
# figure out what we're going to do with stdio
|
||||
if not daemon:
|
||||
# do nothing, this process is open until the spawns close anyway
|
||||
re_out = None
|
||||
re_err = None
|
||||
else:
|
||||
re_err = subprocess.STDOUT
|
||||
if wait:
|
||||
# we're going to need to block on this...
|
||||
re_out = subprocess.PIPE
|
||||
else:
|
||||
re_out = open(os.devnull, 'w+b')
|
||||
proc = subprocess.Popen(args, stdout=re_out, stderr=re_err)
|
||||
pid_file = self.get_pid_file_name(conf_file)
|
||||
write_file(pid_file, proc.pid)
|
||||
self.procs.append(proc)
|
||||
return proc.pid
|
||||
|
||||
def wait(self, **kwargs):
|
||||
"""
|
||||
wait on spawned procs to start
|
||||
"""
|
||||
status = 0
|
||||
for proc in self.procs:
|
||||
# wait for process to close it's stdout
|
||||
output = proc.stdout.read()
|
||||
if output:
|
||||
print output
|
||||
proc.communicate()
|
||||
if proc.returncode:
|
||||
status += 1
|
||||
return status
|
||||
|
||||
def interact(self, **kwargs):
|
||||
"""
|
||||
wait on spawned procs to terminate
|
||||
"""
|
||||
status = 0
|
||||
for proc in self.procs:
|
||||
# wait for process to terminate
|
||||
proc.communicate()
|
||||
if proc.returncode:
|
||||
status += 1
|
||||
return status
|
||||
|
||||
def launch(self, **kwargs):
|
||||
"""
|
||||
Collect conf files and attempt to spawn the processes for this server
|
||||
"""
|
||||
conf_files = self.conf_files(**kwargs)
|
||||
if not conf_files:
|
||||
return []
|
||||
|
||||
pids = self.get_running_pids(**kwargs)
|
||||
|
||||
already_started = False
|
||||
for pid, pid_file in pids.items():
|
||||
conf_file = self.get_conf_file_name(pid_file)
|
||||
# for legacy compat you can't start other servers if one server is
|
||||
# already running (unless -n specifies which one you want), this
|
||||
# restriction could potentially be lifted, and launch could start
|
||||
# any unstarted instances
|
||||
if conf_file in conf_files:
|
||||
already_started = True
|
||||
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
||||
elif not kwargs.get('number', 0):
|
||||
already_started = True
|
||||
print _("%s running (%s - %s)") % (self.server, pid, pid_file)
|
||||
|
||||
if already_started:
|
||||
print _("%s already started...") % self.server
|
||||
return []
|
||||
|
||||
if self.server not in START_ONCE_SERVERS:
|
||||
kwargs['once'] = False
|
||||
|
||||
pids = {}
|
||||
for conf_file in conf_files:
|
||||
if kwargs.get('once'):
|
||||
msg = _('Running %s once') % self.server
|
||||
else:
|
||||
msg = _('Starting %s') % self.server
|
||||
print '%s...(%s)' % (msg, conf_file)
|
||||
try:
|
||||
pid = self.spawn(conf_file, **kwargs)
|
||||
except OSError, e:
|
||||
if e.errno == errno.ENOENT:
|
||||
# TODO: should I check if self.cmd exists earlier?
|
||||
print _("%s does not exist") % self.cmd
|
||||
break
|
||||
pids[pid] = conf_file
|
||||
|
||||
return pids
|
||||
|
||||
def stop(self, **kwargs):
|
||||
"""Send stop signals to pids for this server
|
||||
|
||||
:returns: a dict mapping pids (ints) to pid_files (paths)
|
||||
|
||||
"""
|
||||
return self.kill_running_pids(**kwargs)
|
||||
@ -34,6 +34,7 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError
|
||||
from optparse import OptionParser
|
||||
from tempfile import mkstemp
|
||||
import cPickle as pickle
|
||||
import glob
|
||||
from urlparse import urlparse as stdlib_urlparse, ParseResult
|
||||
|
||||
import eventlet
|
||||
@ -471,7 +472,10 @@ def capture_stdio(logger, **kwargs):
|
||||
stdio_fds = [0, 1, 2]
|
||||
for _junk, handler in getattr(get_logger,
|
||||
'console_handler4logger', {}).items():
|
||||
stdio_fds.remove(handler.stream.fileno())
|
||||
try:
|
||||
stdio_fds.remove(handler.stream.fileno())
|
||||
except ValueError:
|
||||
pass # fd not in list
|
||||
|
||||
with open(os.devnull, 'r+b') as nullfile:
|
||||
# close stdio (excludes fds open for logging)
|
||||
@ -786,6 +790,60 @@ def write_pickle(obj, dest, tmp):
|
||||
renamer(tmppath, dest)
|
||||
|
||||
|
||||
def search_tree(root, glob_match, ext):
|
||||
"""Look in root, for any files/dirs matching glob, recurively traversing
|
||||
any found directories looking for files ending with ext
|
||||
|
||||
:param root: start of search path
|
||||
:param glob_match: glob to match in root, matching dirs are traversed with
|
||||
os.walk
|
||||
:param ext: only files that end in ext will be returned
|
||||
|
||||
:returns: list of full paths to matching files, sorted
|
||||
|
||||
"""
|
||||
found_files = []
|
||||
for path in glob.glob(os.path.join(root, glob_match)):
|
||||
if path.endswith(ext):
|
||||
found_files.append(path)
|
||||
else:
|
||||
for root, dirs, files in os.walk(path):
|
||||
for file in files:
|
||||
if file.endswith(ext):
|
||||
found_files.append(os.path.join(root, file))
|
||||
return sorted(found_files)
|
||||
|
||||
|
||||
def write_file(path, contents):
|
||||
"""Write contents to file at path
|
||||
|
||||
:param path: any path, subdirs will be created as needed
|
||||
:param contents: data to write to file, will be converted to string
|
||||
|
||||
"""
|
||||
dirname, name = os.path.split(path)
|
||||
if not os.path.exists(dirname):
|
||||
try:
|
||||
os.makedirs(dirname)
|
||||
except OSError, err:
|
||||
if err.errno == errno.EACCES:
|
||||
sys.exit('Unable to create %s. Running as '
|
||||
'non-root?' % dirname)
|
||||
with open(path, 'w') as f:
|
||||
f.write('%s' % contents)
|
||||
|
||||
|
||||
def remove_file(path):
|
||||
"""Quiet wrapper for os.unlink, OSErrors are suppressed
|
||||
|
||||
:param path: first and only argument passed to os.unlink
|
||||
"""
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def audit_location_generator(devices, datadir, mount_check=True, logger=None):
|
||||
'''
|
||||
Given a devices path and a data directory, yield (path, device,
|
||||
|
||||
@ -119,8 +119,6 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
|
||||
logger = get_logger(conf, log_name,
|
||||
log_to_console=kwargs.pop('verbose', False), log_route='wsgi')
|
||||
|
||||
# redirect errors to logger and close stdio
|
||||
capture_stdio(logger)
|
||||
# bind to address and port
|
||||
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
|
||||
# remaining tasks should not require elevated privileges
|
||||
@ -129,6 +127,9 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
|
||||
# finally after binding to ports and privilege drop, run app __init__ code
|
||||
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
|
||||
|
||||
# redirect errors to logger and close stdio
|
||||
capture_stdio(logger)
|
||||
|
||||
def run_server():
|
||||
wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
|
||||
eventlet.hubs.use_hub('poll')
|
||||
|
||||
@ -4,6 +4,8 @@ import os
|
||||
from contextlib import contextmanager
|
||||
from tempfile import NamedTemporaryFile
|
||||
from eventlet.green import socket
|
||||
from tempfile import mkdtemp
|
||||
from shutil import rmtree
|
||||
|
||||
|
||||
def readuntil2crlfs(fd):
|
||||
@ -68,6 +70,27 @@ xattr.setxattr = _setxattr
|
||||
xattr.getxattr = _getxattr
|
||||
|
||||
|
||||
@contextmanager
|
||||
def temptree(files, contents=''):
|
||||
# generate enough contents to fill the files
|
||||
c = len(files)
|
||||
contents = (list(contents) + [''] * c)[:c]
|
||||
tempdir = mkdtemp()
|
||||
for path, content in zip(files, contents):
|
||||
if os.path.isabs(path):
|
||||
path = '.' + path
|
||||
new_path = os.path.join(tempdir, path)
|
||||
subdir = os.path.dirname(new_path)
|
||||
if not os.path.exists(subdir):
|
||||
os.makedirs(subdir)
|
||||
with open(new_path, 'w') as f:
|
||||
f.write(str(content))
|
||||
try:
|
||||
yield tempdir
|
||||
finally:
|
||||
rmtree(tempdir)
|
||||
|
||||
|
||||
class MockTrue(object):
|
||||
"""
|
||||
Instances of MockTrue evaluate like True
|
||||
|
||||
1636
test/unit/common/test_manager.py
Normal file
1636
test/unit/common/test_manager.py
Normal file
File diff suppressed because it is too large
Load Diff
@ -16,6 +16,7 @@
|
||||
""" Tests for swift.common.utils """
|
||||
|
||||
from __future__ import with_statement
|
||||
from test.unit import temptree
|
||||
import logging
|
||||
import mimetools
|
||||
import os
|
||||
@ -660,5 +661,86 @@ log_name = yarr'''
|
||||
self.assertTrue(abs(100 - (time.time() - start) * 100) < 10)
|
||||
|
||||
|
||||
def test_search_tree(self):
|
||||
# file match & ext miss
|
||||
with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t:
|
||||
asdf = utils.search_tree(t, 'a*', '.conf')
|
||||
self.assertEquals(len(asdf), 1)
|
||||
self.assertEquals(asdf[0],
|
||||
os.path.join(t, 'asdf.conf'))
|
||||
|
||||
# multi-file match & glob miss & sort
|
||||
with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t:
|
||||
app_bins = utils.search_tree(t, 'app*', 'bin')
|
||||
self.assertEquals(len(app_bins), 2)
|
||||
self.assertEquals(app_bins[0],
|
||||
os.path.join(t, 'apple.bin'))
|
||||
self.assertEquals(app_bins[1],
|
||||
os.path.join(t, 'application.bin'))
|
||||
|
||||
# test file in folder & ext miss & glob miss
|
||||
files = (
|
||||
'sub/file1.ini',
|
||||
'sub/file2.conf',
|
||||
'sub.bin',
|
||||
'bus.ini',
|
||||
'bus/file3.ini',
|
||||
)
|
||||
with temptree(files) as t:
|
||||
sub_ini = utils.search_tree(t, 'sub*', '.ini')
|
||||
self.assertEquals(len(sub_ini), 1)
|
||||
self.assertEquals(sub_ini[0],
|
||||
os.path.join(t, 'sub/file1.ini'))
|
||||
|
||||
# test multi-file in folder & sub-folder & ext miss & glob miss
|
||||
files = (
|
||||
'folder_file.txt',
|
||||
'folder/1.txt',
|
||||
'folder/sub/2.txt',
|
||||
'folder2/3.txt',
|
||||
'Folder3/4.txt'
|
||||
'folder.rc',
|
||||
)
|
||||
with temptree(files) as t:
|
||||
folder_texts = utils.search_tree(t, 'folder*', '.txt')
|
||||
self.assertEquals(len(folder_texts), 4)
|
||||
f1 = os.path.join(t, 'folder_file.txt')
|
||||
f2 = os.path.join(t, 'folder/1.txt')
|
||||
f3 = os.path.join(t, 'folder/sub/2.txt')
|
||||
f4 = os.path.join(t, 'folder2/3.txt')
|
||||
for f in [f1, f2, f3, f4]:
|
||||
self.assert_(f in folder_texts)
|
||||
|
||||
def test_write_file(self):
|
||||
with temptree([]) as t:
|
||||
file_name = os.path.join(t, 'test')
|
||||
utils.write_file(file_name, 'test')
|
||||
with open(file_name, 'r') as f:
|
||||
contents = f.read()
|
||||
self.assertEquals(contents, 'test')
|
||||
# and also subdirs
|
||||
file_name = os.path.join(t, 'subdir/test2')
|
||||
utils.write_file(file_name, 'test2')
|
||||
with open(file_name, 'r') as f:
|
||||
contents = f.read()
|
||||
self.assertEquals(contents, 'test2')
|
||||
# but can't over-write files
|
||||
file_name = os.path.join(t, 'subdir/test2/test3')
|
||||
self.assertRaises(IOError, utils.write_file, file_name,
|
||||
'test3')
|
||||
|
||||
def test_remove_file(self):
|
||||
with temptree([]) as t:
|
||||
file_name = os.path.join(t, 'blah.pid')
|
||||
# assert no raise
|
||||
self.assertEquals(os.path.exists(file_name), False)
|
||||
self.assertEquals(utils.remove_file(file_name), None)
|
||||
with open(file_name, 'w') as f:
|
||||
f.write('1')
|
||||
self.assert_(os.path.exists(file_name))
|
||||
self.assertEquals(utils.remove_file(file_name), None)
|
||||
self.assertFalse(os.path.exists(file_name))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user