Add targeted config loading to swift-init

This allows an easier and more explicit way to tell swift-init to run on
specific servers.  For example with an SAIO, this allows you to do
something like:

swift-init object-server.1 reload

to reload just the 1st object server.  A more real world example is when
you are running separate servers for replication.  In this example you
might have an object-server/public.conf and
object-server/replication.conf.  With this change you can do something
like:

swift-init object-server.replication reload

to just reload the replication server.

DocImpact
Change-Id: I5c6046b5ee28e17dadfc5fc53d1d872d9bb8fe48

bin/swift-init

@@ -20,7 +20,7 @@ from optparse import OptionParser
 from swift.common.manager import Manager, UnknownCommandError, \
     KILL_WAIT, RUN_DIR
 
-USAGE = """%prog <server> [<server> ...] <command> [options]
+USAGE = """%prog <server>[.config] [<server>[.config] ...] <command> [options]
 
 Commands:
 """ + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])

doc/source/admin_guide.rst

@@ -1063,6 +1063,14 @@ A graceful shutdown or reload will finish any current requests before
 completely stopping the old service.  There is also a special case of 
 `swift-init all <command>`, which will run the command for all swift services.
 
+In cases where there are multiple configs for a service, a specific config
+can be managed with ``swift-init <service>.<config> <command>``.
+For example, when a separate replication network is used, there might be
+`/etc/swift/object-server/public.conf` for the object server and
+`/etc/swift/object-server/replication.conf` for the replication services.
+In this case, the replication services could be restarted with
+``swift-init object-server.replication restart``.
+
 --------------
 Object Auditor
 --------------

swift/common/manager.py

@@ -144,23 +144,24 @@ class Manager(object):
     """
 
     def __init__(self, servers, run_dir=RUN_DIR):
-        server_names = set()
+        self.server_names = set()
         for server in servers:
             if server == 'all':
-                server_names.update(ALL_SERVERS)
+                self.server_names.update(ALL_SERVERS)
             elif server == 'main':
-                server_names.update(MAIN_SERVERS)
+                self.server_names.update(MAIN_SERVERS)
             elif server == 'rest':
-                server_names.update(REST_SERVERS)
+                self.server_names.update(REST_SERVERS)
             elif '*' in server:
                 # convert glob to regex
-                server_names.update([s for s in ALL_SERVERS if
+                self.server_names.update([
-                                     re.match(server.replace('*', '.*'), s)])
+                    s for s in ALL_SERVERS if
+                    re.match(server.replace('*', '.*'), s)])
             else:
-                server_names.add(server)
+                self.server_names.add(server)
 
         self.servers = set()
-        for name in server_names:
+        for name in self.server_names:
             self.servers.add(Server(name, run_dir))
 
     def __iter__(self):
@@ -288,8 +289,8 @@ class Manager(object):
         """
         kwargs['graceful'] = True
         status = 0
-        for server in self.servers:
+        for server in self.server_names:
-            m = Manager([server.server])
+            m = Manager([server])
             status += m.stop(**kwargs)
             status += m.start(**kwargs)
         return status
@@ -345,11 +346,15 @@ class Server(object):
     """
 
     def __init__(self, server, run_dir=RUN_DIR):
-        if '-' not in server:
-            server = '%s-server' % server
         self.server = server.lower()
-        self.type = server.rsplit('-', 1)[0]
+        if '.' in self.server:
-        self.cmd = 'swift-%s' % server
+            self.server, self.conf = self.server.rsplit('.', 1)
+        else:
+            self.conf = None
+        if '-' not in self.server:
+            self.server = '%s-server' % self.server
+        self.type = self.server.rsplit('-', 1)[0]
+        self.cmd = 'swift-%s' % self.server
         self.procs = []
         self.run_dir = run_dir
 
@@ -407,10 +412,15 @@ class Server(object):
         :returns: list of conf files
         """
         if self.server in STANDALONE_SERVERS:
-            found_conf_files = search_tree(SWIFT_DIR, self.server + '*',
+            server_search = self.server
-                                           '.conf', dir_ext='.conf.d')
         else:
-            found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type,
+            server_search = "%s-server" % self.type
+        if self.conf is not None:
+            found_conf_files = search_tree(SWIFT_DIR, server_search,
+                                           self.conf + '.conf',
+                                           dir_ext=self.conf + '.conf.d')
+        else:
+            found_conf_files = search_tree(SWIFT_DIR, server_search + '*',
                                            '.conf', dir_ext='.conf.d')
         number = kwargs.get('number')
         if number:
@@ -440,7 +450,12 @@ class Server(object):
 
         :returns: list of pid files
         """
-        pid_files = search_tree(self.run_dir, '%s*' % self.server)
+        if self.conf is not None:
+            pid_files = search_tree(self.run_dir, '%s*' % self.server,
+                                    exts=[self.conf + '.pid',
+                                          self.conf + '.pid.d'])
+        else:
+            pid_files = search_tree(self.run_dir, '%s*' % self.server)
         if kwargs.get('number', 0):
             conf_files = self.conf_files(**kwargs)
             # filter pid_files to match the index of numbered conf_file

swift/common/utils.py

@@ -1644,7 +1644,7 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
         renamer(tmppath, dest)
 
 
-def search_tree(root, glob_match, ext='', dir_ext=None):
+def search_tree(root, glob_match, ext='', exts=None, dir_ext=None):
     """Look in root, for any files/dirs matching glob, recursively traversing
     any found directories looking for files ending with ext
 
@@ -1658,6 +1658,7 @@ def search_tree(root, glob_match, ext='', dir_ext=None):
     :returns: list of full paths to matching files, sorted
 
     """
+    exts = exts or [ext]
     found_files = []
     for path in glob.glob(os.path.join(root, glob_match)):
         if os.path.isdir(path):
@@ -1667,7 +1668,7 @@ def search_tree(root, glob_match, ext='', dir_ext=None):
                     # the root is a config dir, descend no further
                     break
                 for file_ in files:
-                    if ext and not file_.endswith(ext):
+                    if any(exts) and not any(file_.endswith(e) for e in exts):
                         continue
                     found_files.append(os.path.join(root, file_))
                 found_dir = False

test/unit/common/test_manager.py

@@ -412,6 +412,22 @@ class TestServer(unittest.TestCase):
             conf_files = server.conf_files(number=5)
             self.assertFalse(conf_files)
 
+        # test geting specific conf
+        conf_files = (
+            'account-server/1.conf',
+            'account-server/2.conf',
+            'account-server/3.conf',
+            'account-server/4.conf',
+        )
+        with temptree(conf_files) as t:
+            manager.SWIFT_DIR = t
+            server = manager.Server('account.2')
+            conf_files = server.conf_files()
+            self.assertEquals(len(conf_files), 1)
+            conf_file = conf_files[0]
+            self.assertEquals(conf_file,
+                              self.join_swift_dir('account-server/2.conf'))
+
         # test verbose & quiet
         conf_files = (
             'auth-server.ini',
@@ -471,6 +487,32 @@ class TestServer(unittest.TestCase):
             proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d')
             self.assertEquals(proxy_conf_dir, conf_dir)
 
+    def test_named_conf_dir(self):
+        conf_files = (
+            'object-server/base.conf-template',
+            'object-server/object-server.conf.d/00_base.conf',
+            'object-server/object-server.conf.d/10_server.conf',
+            'object-server/object-replication.conf.d/00_base.conf',
+            'object-server/object-replication.conf.d/10_server.conf',
+        )
+        with temptree(conf_files) as t:
+            manager.SWIFT_DIR = t
+            server = manager.Server('object.replication')
+            conf_dirs = server.conf_files()
+            self.assertEquals(len(conf_dirs), 1)
+            conf_dir = conf_dirs[0]
+            replication_server_conf_dir = self.join_swift_dir(
+                'object-server/object-replication.conf.d')
+            self.assertEquals(replication_server_conf_dir, conf_dir)
+            # and again with no named filter
+            server = manager.Server('object')
+            conf_dirs = server.conf_files()
+            self.assertEquals(len(conf_dirs), 2)
+            for named_conf in ('server', 'replication'):
+                conf_dir = self.join_swift_dir(
+                    'object-server/object-%s.conf.d' % named_conf)
+                self.assert_(conf_dir in conf_dirs)
+
     def test_conf_dir(self):
         conf_files = (
             'object-server/object-server.conf-base',
@@ -498,6 +540,29 @@ class TestServer(unittest.TestCase):
             sorted_confs = sorted([c1, c2, c3, c4])
             self.assertEquals(conf_dirs, sorted_confs)
 
+    def test_named_conf_dir_pid_files(self):
+        conf_files = (
+            'object-server/object-server.pid.d',
+            'object-server/object-replication.pid.d',
+        )
+        with temptree(conf_files) as t:
+            manager.RUN_DIR = t
+            server = manager.Server('object.replication', run_dir=t)
+            pid_files = server.pid_files()
+            self.assertEquals(len(pid_files), 1)
+            pid_file = pid_files[0]
+            replication_server_pid = self.join_run_dir(
+                'object-server/object-replication.pid.d')
+            self.assertEquals(replication_server_pid, pid_file)
+            # and again with no named filter
+            server = manager.Server('object', run_dir=t)
+            pid_files = server.pid_files()
+            self.assertEquals(len(pid_files), 2)
+            for named_pid in ('server', 'replication'):
+                pid_file = self.join_run_dir(
+                    'object-server/object-%s.pid.d' % named_pid)
+                self.assert_(pid_file in pid_files)
+
     def test_iter_pid_files(self):
         """
         Server.iter_pid_files is kinda boring, test the
@@ -581,6 +646,34 @@ class TestServer(unittest.TestCase):
                 pids = list(server.iter_pid_files(number=5))
                 self.assertFalse(pids)
 
+        # test get pid_files by conf name
+        conf_files = (
+            'object-server/1.conf',
+            'object-server/2.conf',
+            'object-server/3.conf',
+            'object-server/4.conf',
+        )
+
+        pid_files = (
+            ('object-server/1.pid', 1),
+            ('object-server/2.pid', 2),
+            ('object-server/5.pid', 5),
+        )
+
+        with temptree(conf_files) as swift_dir:
+            manager.SWIFT_DIR = swift_dir
+            files, pids = zip(*pid_files)
+            with temptree(files, pids) as t:
+                manager.RUN_DIR = t
+                server = manager.Server('object.2', run_dir=t)
+                # test get pid with matching conf
+                pids = list(server.iter_pid_files())
+                self.assertEquals(len(pids), 1)
+                pid_file, pid = pids[0]
+                self.assertEquals(pid, 2)
+                pid_two = self.join_run_dir('object-server/2.pid')
+                self.assertEquals(pid_file, pid_two)
+
     def test_signal_pids(self):
         pid_files = (
             ('proxy-server.pid', 1),

test/unit/common/test_utils.py

@@ -1242,6 +1242,34 @@ log_name = %(yarr)s'''
             conf_dir = os.path.join(t, 'object-server/%d.conf.d' % (i + 1))
             self.assert_(conf_dir in conf_dirs)
 
+    def test_search_tree_conf_dir_with_named_conf_match(self):
+        files = (
+            'proxy-server/proxy-server.conf.d/base.conf',
+            'proxy-server/proxy-server.conf.d/pipeline.conf',
+            'proxy-server/proxy-noauth.conf.d/base.conf',
+            'proxy-server/proxy-noauth.conf.d/pipeline.conf',
+        )
+        with temptree(files) as t:
+            conf_dirs = utils.search_tree(t, 'proxy-server', 'noauth.conf',
+                                          dir_ext='noauth.conf.d')
+        self.assertEquals(len(conf_dirs), 1)
+        conf_dir = conf_dirs[0]
+        expected = os.path.join(t, 'proxy-server/proxy-noauth.conf.d')
+        self.assertEqual(conf_dir, expected)
+
+    def test_search_tree_conf_dir_pid_with_named_conf_match(self):
+        files = (
+            'proxy-server/proxy-server.pid.d',
+            'proxy-server/proxy-noauth.pid.d',
+        )
+        with temptree(files) as t:
+            pid_files = utils.search_tree(t, 'proxy-server',
+                                          exts=['noauth.pid', 'noauth.pid.d'])
+        self.assertEquals(len(pid_files), 1)
+        pid_file = pid_files[0]
+        expected = os.path.join(t, 'proxy-server/proxy-noauth.pid.d')
+        self.assertEqual(pid_file, expected)
+
     def test_write_file(self):
         with temptree([]) as t:
             file_name = os.path.join(t, 'test')