Support excludes when recursively processing paths

Include an additional option to exclude some paths/patterns from being
included in the search to allow a complex hierarchy of directories to be
referenced under a single path. Provide support to allow the user to
specify patterns, relative and/or absolute path references.

Implements support for excluding based on absolute path, relative path
and based on simple shell globing patterns.

Change-Id: I236b364c268fd2bf5511a2c6d9a1c87914f3b086

doc/source/installation.rst

@@ -74,6 +74,14 @@ job_builder section
   (Optional) If set to True, jenkins job builder will search for job
   definition files recursively
 
+**exclude**
+  (Optional) If set to a list of values separated by ':', these paths will be
+  excluded from the list of paths to be processed when searching recursively.
+  Values containing no ``/`` will be matched against directory names at all
+  levels, those starting with ``/`` will be considered absolute, while others
+  containing a ``/`` somewhere other than the start of the value will be
+  considered relative to the starting path.
+
 **allow_duplicates**
   (Optional) By default `jenkins-jobs` will abort any time a duplicate macro,
   template, job-group or job name is encountered as it cannot establish the
@@ -155,6 +163,59 @@ Jenkins instances suited to various needs you may want to share configuration
 between those instances (global). Furthermore, there may be various ways you
 would like to structure jobs within a given instance.
 
+Recursive Searching of Paths
+----------------------------
+
+In addition to passing multiple paths to JJB it is also possible to enable
+recursive searching to process all yaml files in the tree beneath each path.
+For example::
+
+  For a tree:
+    /path/
+      to/
+        defs/
+          ci_jobs/
+          release_jobs/
+        globals/
+          macros/
+          templates/
+
+  jenkins-jobs update -r /path/to/defs:/path/to/globals
+
+JJB will search defs/ci_jobs, defs/release_jobs, globals/macros and
+globals/templates in addition to the defs and globals trees.
+
+Excluding Paths
+---------------
+
+To allow a complex tree of jobs where some jobs are managed differently without
+needing to explicitly provide each path, the recursive path processing supports
+excluding paths based on absolute paths, relative paths and patterns. For
+example::
+
+  For a tree:
+    /path/
+      to/
+        defs/
+          ci_jobs/
+            manual/
+          release_jobs/
+            manual/
+          qa_jobs/
+        globals/
+          macros/
+          templates/
+          special/
+
+  jenkins-jobs update -r -x man*:./qa_jobs -x /path/to/defs/globals/special \
+    /path/to/defs:/path/to/globals
+
+JJB search the given paths, ignoring the directories qa_jobs, ci_jobs/manual,
+release_jobs/manual, and globals/special when building the list of yaml files
+to be processed. Absolute paths are denoted by starting from the root,
+relative by containing the path separator, and patterns by having neither.
+Patterns use simple shell globing to match directories.
+
 Deleting Jobs
 ^^^^^^^^^^^^^
 Jenkins Job Builder supports deleting jobs from Jenkins.

etc/jenkins_jobs.ini-sample

@@ -3,6 +3,7 @@ ignore_cache=True
 keep_descriptions=False
 include_path=.:scripts:~/git/
 recursive=False
+exclude=.*:manual:./development
 allow_duplicates=False
 
 [jenkins]

jenkins_jobs/cmd.py

@@ -15,6 +15,7 @@
 
 import argparse
 from six.moves import configparser, StringIO
+import fnmatch
 import logging
 import os
 import platform
@@ -33,6 +34,7 @@ DEFAULT_CONF = """
 keep_descriptions=False
 ignore_cache=False
 recursive=False
+exclude=.*
 allow_duplicates=False
 
 [jenkins]
@@ -51,11 +53,28 @@ def confirm(question):
         sys.exit('Aborted')
 
 
-def recurse_path(root):
+def recurse_path(root, excludes=None):
+    if excludes is None:
+        excludes = []
+
     basepath = os.path.realpath(root)
     pathlist = [basepath]
 
+    patterns = [e for e in excludes if os.path.sep not in e]
+    absolute = [e for e in excludes if os.path.isabs(e)]
+    relative = [e for e in excludes if os.path.sep in e and
+                not os.path.isabs(e)]
     for root, dirs, files in os.walk(basepath, topdown=True):
+        dirs[:] = [
+            d for d in dirs
+            if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])
+            if not any([fnmatch.fnmatch(os.path.abspath(os.path.join(root, d)),
+                                        path)
+                        for path in absolute])
+            if not any([fnmatch.fnmatch(os.path.relpath(os.path.join(root, d)),
+                                        path)
+                        for path in relative])
+        ]
         pathlist.extend([os.path.join(root, path) for path in dirs])
 
     return pathlist
@@ -68,6 +87,10 @@ def create_parser():
     recursive_parser.add_argument('-r', '--recursive', action='store_true',
                                   dest='recursive', default=False,
                                   help='look for yaml files recursively')
+    recursive_parser.add_argument('-x', '--exclude', dest='exclude',
+                                  action='append', default=[],
+                                  help='paths to exclude when using recursive'
+                                       ' search, uses standard globbing.')
     subparser = parser.add_subparsers(help='update, test or delete job',
                                       dest='command')
 
@@ -95,7 +118,7 @@ def create_parser():
 
     # subparser: delete
 
-    parser_delete = subparser.add_parser('delete')
+    parser_delete = subparser.add_parser('delete', parents=[recursive_parser])
     parser_delete.add_argument('name', help='name of job', nargs='+')
     parser_delete.add_argument('-p', '--path', default=None,
                                help='colon-separated list of paths to'
@@ -233,10 +256,13 @@ def execute(options, config):
         do_recurse = (getattr(options, 'recursive', False) or
                       config.getboolean('job_builder', 'recursive'))
 
+        excludes = [e for elist in options.exclude
+                    for e in elist.split(os.pathsep)] or \
+            config.get('job_builder', 'exclude').split(os.pathsep)
         paths = []
         for path in options.path:
             if do_recurse and os.path.isdir(path):
-                paths.extend(recurse_path(path))
+                paths.extend(recurse_path(path, excludes))
             else:
                 paths.append(path)
         options.path = paths

tests/cmd/subcommands/test_test.py

@@ -7,40 +7,41 @@ import jenkins
 
 from jenkins_jobs import cmd
 from jenkins_jobs.errors import JenkinsJobsException
-from tests.cmd.test_cmd import CmdTestsBase
 from tests.base import mock
+from tests.cmd.test_cmd import CmdTestsBase
+from tests.cmd.test_recurse_path import fake_os_walk
 
 
 os_walk_return_values = {
     '/jjb_projects': [
-        ('/jjb_projects', ('dir1', 'dir2', 'dir3'), ()),
+        ('/jjb_projects', (['dir1', 'dir2', 'dir3'], ())),
-        ('/jjb_projects/dir1', ('bar',), ()),
+        ('/jjb_projects/dir1', (['bar'], ())),
-        ('/jjb_projects/dir2', ('baz',), ()),
+        ('/jjb_projects/dir2', (['baz'], ())),
-        ('/jjb_projects/dir3', (), ()),
+        ('/jjb_projects/dir3', ([], ())),
-        ('/jjb_projects/dir1/bar', (), ()),
+        ('/jjb_projects/dir1/bar', ([], ())),
-        ('/jjb_projects/dir2/baz', (), ()),
+        ('/jjb_projects/dir2/baz', ([], ())),
     ],
     '/jjb_templates': [
-        ('/jjb_templates', ('dir1', 'dir2', 'dir3'), ()),
+        ('/jjb_templates', (['dir1', 'dir2', 'dir3'], ())),
-        ('/jjb_templates/dir1', ('bar',), ()),
+        ('/jjb_templates/dir1', (['bar'], ())),
-        ('/jjb_templates/dir2', ('baz',), ()),
+        ('/jjb_templates/dir2', (['baz'], ())),
-        ('/jjb_templates/dir3', (), ()),
+        ('/jjb_templates/dir3', ([], ())),
-        ('/jjb_templates/dir1/bar', (), ()),
+        ('/jjb_templates/dir1/bar', ([], ())),
-        ('/jjb_templates/dir2/baz', (), ()),
+        ('/jjb_templates/dir2/baz', ([], ())),
     ],
     '/jjb_macros': [
-        ('/jjb_macros', ('dir1', 'dir2', 'dir3'), ()),
+        ('/jjb_macros', (['dir1', 'dir2', 'dir3'], ())),
-        ('/jjb_macros/dir1', ('bar',), ()),
+        ('/jjb_macros/dir1', (['bar'], ())),
-        ('/jjb_macros/dir2', ('baz',), ()),
+        ('/jjb_macros/dir2', (['baz'], ())),
-        ('/jjb_macros/dir3', (), ()),
+        ('/jjb_macros/dir3', ([], ())),
-        ('/jjb_macros/dir1/bar', (), ()),
+        ('/jjb_macros/dir1/bar', ([], ())),
-        ('/jjb_macros/dir2/baz', (), ()),
+        ('/jjb_macros/dir2/baz', ([], ())),
     ],
 }
 
 
 def os_walk_side_effects(path_name, topdown):
-    return os_walk_return_values[path_name]
+    return fake_os_walk(os_walk_return_values[path_name])(path_name, topdown)
 
 
 @mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
@@ -114,8 +115,7 @@ class TestTests(CmdTestsBase):
         path_list = os_walk_return_values.keys()
         paths = []
         for path in path_list:
-            paths.extend([p for p, _, _ in
+            paths.extend([p for p, _ in os_walk_return_values[path]])
-                          os_walk_return_values[path]])
 
         multipath = os.pathsep.join(path_list)
 
@@ -132,6 +132,34 @@ class TestTests(CmdTestsBase):
 
         update_job_mock.assert_called_with(paths, [], output=args.output_dir)
 
+    @mock.patch('jenkins_jobs.cmd.Builder.update_job')
+    @mock.patch('jenkins_jobs.cmd.os.path.isdir')
+    @mock.patch('jenkins_jobs.cmd.os.walk')
+    def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock,
+                                                update_job_mock):
+        """
+        Run test mode and pass multiple paths with recursive path option.
+        """
+
+        os_walk_mock.side_effect = os_walk_side_effects
+        isdir_mock.return_value = True
+
+        path_list = os_walk_return_values.keys()
+        paths = []
+        for path in path_list:
+            paths.extend([p for p, __ in os_walk_return_values[path]
+                          if 'dir1' not in p and 'dir2' not in p])
+
+        multipath = os.pathsep.join(path_list)
+
+        args = self.parser.parse_args(['test', '-r', multipath, '-x',
+                                       'dir1:dir2'])
+        args.output_dir = mock.MagicMock()
+
+        cmd.execute(args, self.config)
+
+        update_job_mock.assert_called_with(paths, [], output=args.output_dir)
+
     def test_console_output(self):
         """
         Run test mode and verify that resulting XML gets sent to the console.

tests/cmd/test_recurse_path.py

@@ -0,0 +1,136 @@
+import os
+
+import mock
+import testtools
+
+from jenkins_jobs import cmd
+
+
+def fake_os_walk(paths):
+    """Helper function for mocking os.walk() where must test that manipulation
+    of the returned dirs variable works as expected
+    """
+    paths_dict = dict(paths)
+
+    def os_walk(top, topdown=True):
+        dirs, nondirs = paths_dict[top]
+        yield top, dirs, nondirs
+        for name in dirs:
+            # hard code use of '/' to ensure the test data can be defined as
+            # simple strings otherwise tests using this helper will break on
+            # platforms where os.path.sep is different.
+            new_path = "/".join([top, name])
+            for x in os_walk(new_path, topdown):
+                yield x
+    return os_walk
+
+
+# Testing the cmd module can sometimes result in the CacheStorage class
+# attempting to create the cache directory multiple times as the tests
+# are run in parallel.  Stub out the CacheStorage to ensure that each
+# test can safely create the object without effect.
+@mock.patch('jenkins_jobs.builder.CacheStorage', mock.MagicMock)
+class CmdRecursePath(testtools.TestCase):
+
+    @mock.patch('jenkins_jobs.cmd.os.walk')
+    def test_recursive_path_option_exclude_pattern(self, oswalk_mock):
+        """
+        Test paths returned by the recursive processing when using pattern
+        excludes.
+
+        testing paths
+            /jjb_configs/dir1/test1/
+            /jjb_configs/dir1/file
+            /jjb_configs/dir2/test2/
+            /jjb_configs/dir3/bar/
+            /jjb_configs/test3/bar/
+            /jjb_configs/test3/baz/
+        """
+
+        os_walk_paths = [
+            ('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
+            ('/jjb_configs/dir1', (['test1'], ('file'))),
+            ('/jjb_configs/dir2', (['test2'], ())),
+            ('/jjb_configs/dir3', (['bar'], ())),
+            ('/jjb_configs/dir3/bar', ([], ())),
+            ('/jjb_configs/test3/bar', None),
+            ('/jjb_configs/test3/baz', None)
+        ]
+
+        paths = [k for k, v in os_walk_paths if v is not None]
+
+        oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
+        self.assertEqual(paths, cmd.recurse_path('/jjb_configs', ['test*']))
+
+    @mock.patch('jenkins_jobs.cmd.os.walk')
+    def test_recursive_path_option_exclude_absolute(self, oswalk_mock):
+        """
+        Test paths returned by the recursive processing when using absolute
+        excludes.
+
+        testing paths
+            /jjb_configs/dir1/test1/
+            /jjb_configs/dir1/file
+            /jjb_configs/dir2/test2/
+            /jjb_configs/dir3/bar/
+            /jjb_configs/test3/bar/
+            /jjb_configs/test3/baz/
+        """
+
+        os_walk_paths = [
+            ('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
+            ('/jjb_configs/dir1', None),
+            ('/jjb_configs/dir2', (['test2'], ())),
+            ('/jjb_configs/dir3', (['bar'], ())),
+            ('/jjb_configs/test3', (['bar', 'baz'], ())),
+            ('/jjb_configs/dir2/test2', ([], ())),
+            ('/jjb_configs/dir3/bar', ([], ())),
+            ('/jjb_configs/test3/bar', ([], ())),
+            ('/jjb_configs/test3/baz', ([], ()))
+        ]
+
+        paths = [k for k, v in os_walk_paths if v is not None]
+
+        oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
+
+        self.assertEqual(paths, cmd.recurse_path('/jjb_configs',
+                                                 ['/jjb_configs/dir1']))
+
+    @mock.patch('jenkins_jobs.cmd.os.walk')
+    def test_recursive_path_option_exclude_relative(self, oswalk_mock):
+        """
+        Test paths returned by the recursive processing when using relative
+        excludes.
+
+        testing paths
+            ./jjb_configs/dir1/test/
+            ./jjb_configs/dir1/file
+            ./jjb_configs/dir2/test/
+            ./jjb_configs/dir3/bar/
+            ./jjb_configs/test3/bar/
+            ./jjb_configs/test3/baz/
+        """
+
+        os_walk_paths = [
+            ('jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
+            ('jjb_configs/dir1', (['test'], ('file'))),
+            ('jjb_configs/dir2', (['test2'], ())),
+            ('jjb_configs/dir3', (['bar'], ())),
+            ('jjb_configs/test3', (['bar', 'baz'], ())),
+            ('jjb_configs/dir1/test', ([], ())),
+            ('jjb_configs/dir2/test2', ([], ())),
+            ('jjb_configs/dir3/bar', ([], ())),
+            ('jjb_configs/test3/bar', None),
+            ('jjb_configs/test3/baz', ([], ()))
+        ]
+
+        rel_os_walk_paths = [
+            (os.path.abspath(
+                os.path.join(os.path.curdir, k)), v) for k, v in os_walk_paths]
+
+        paths = [k for k, v in rel_os_walk_paths if v is not None]
+
+        oswalk_mock.side_effect = fake_os_walk(rel_os_walk_paths)
+
+        self.assertEqual(paths, cmd.recurse_path('jjb_configs',
+                                                 ['jjb_configs/test3/bar']))