From c5c0eb4f16a245f7886ae7471fed6779ee018b8e Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Mon, 7 Apr 2014 23:46:35 +0100 Subject: [PATCH] Support excludes when recursively processing paths Include an additional option to exclude some paths/patterns from being included in the search to allow a complex hierarchy of directories to be referenced under a single path. Provide support to allow the user to specify patterns, relative and/or absolute path references. Implements support for excluding based on absolute path, relative path and based on simple shell globing patterns. Change-Id: I236b364c268fd2bf5511a2c6d9a1c87914f3b086 --- doc/source/installation.rst | 61 +++++++++++++ etc/jenkins_jobs.ini-sample | 1 + jenkins_jobs/cmd.py | 32 ++++++- tests/cmd/subcommands/test_test.py | 72 ++++++++++----- tests/cmd/test_recurse_path.py | 136 +++++++++++++++++++++++++++++ 5 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 tests/cmd/test_recurse_path.py diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 770d18f16..5c63ee722 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -74,6 +74,14 @@ job_builder section (Optional) If set to True, jenkins job builder will search for job definition files recursively +**exclude** + (Optional) If set to a list of values separated by ':', these paths will be + excluded from the list of paths to be processed when searching recursively. + Values containing no ``/`` will be matched against directory names at all + levels, those starting with ``/`` will be considered absolute, while others + containing a ``/`` somewhere other than the start of the value will be + considered relative to the starting path. + **allow_duplicates** (Optional) By default `jenkins-jobs` will abort any time a duplicate macro, template, job-group or job name is encountered as it cannot establish the @@ -155,6 +163,59 @@ Jenkins instances suited to various needs you may want to share configuration between those instances (global). Furthermore, there may be various ways you would like to structure jobs within a given instance. +Recursive Searching of Paths +---------------------------- + +In addition to passing multiple paths to JJB it is also possible to enable +recursive searching to process all yaml files in the tree beneath each path. +For example:: + + For a tree: + /path/ + to/ + defs/ + ci_jobs/ + release_jobs/ + globals/ + macros/ + templates/ + + jenkins-jobs update -r /path/to/defs:/path/to/globals + +JJB will search defs/ci_jobs, defs/release_jobs, globals/macros and +globals/templates in addition to the defs and globals trees. + +Excluding Paths +--------------- + +To allow a complex tree of jobs where some jobs are managed differently without +needing to explicitly provide each path, the recursive path processing supports +excluding paths based on absolute paths, relative paths and patterns. For +example:: + + For a tree: + /path/ + to/ + defs/ + ci_jobs/ + manual/ + release_jobs/ + manual/ + qa_jobs/ + globals/ + macros/ + templates/ + special/ + + jenkins-jobs update -r -x man*:./qa_jobs -x /path/to/defs/globals/special \ + /path/to/defs:/path/to/globals + +JJB search the given paths, ignoring the directories qa_jobs, ci_jobs/manual, +release_jobs/manual, and globals/special when building the list of yaml files +to be processed. Absolute paths are denoted by starting from the root, +relative by containing the path separator, and patterns by having neither. +Patterns use simple shell globing to match directories. + Deleting Jobs ^^^^^^^^^^^^^ Jenkins Job Builder supports deleting jobs from Jenkins. diff --git a/etc/jenkins_jobs.ini-sample b/etc/jenkins_jobs.ini-sample index 3753c861c..58b670680 100644 --- a/etc/jenkins_jobs.ini-sample +++ b/etc/jenkins_jobs.ini-sample @@ -3,6 +3,7 @@ ignore_cache=True keep_descriptions=False include_path=.:scripts:~/git/ recursive=False +exclude=.*:manual:./development allow_duplicates=False [jenkins] diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index f44fb78f3..780d46244 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -15,6 +15,7 @@ import argparse from six.moves import configparser, StringIO +import fnmatch import logging import os import platform @@ -33,6 +34,7 @@ DEFAULT_CONF = """ keep_descriptions=False ignore_cache=False recursive=False +exclude=.* allow_duplicates=False [jenkins] @@ -51,11 +53,28 @@ def confirm(question): sys.exit('Aborted') -def recurse_path(root): +def recurse_path(root, excludes=None): + if excludes is None: + excludes = [] + basepath = os.path.realpath(root) pathlist = [basepath] + patterns = [e for e in excludes if os.path.sep not in e] + absolute = [e for e in excludes if os.path.isabs(e)] + relative = [e for e in excludes if os.path.sep in e and + not os.path.isabs(e)] for root, dirs, files in os.walk(basepath, topdown=True): + dirs[:] = [ + d for d in dirs + if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns]) + if not any([fnmatch.fnmatch(os.path.abspath(os.path.join(root, d)), + path) + for path in absolute]) + if not any([fnmatch.fnmatch(os.path.relpath(os.path.join(root, d)), + path) + for path in relative]) + ] pathlist.extend([os.path.join(root, path) for path in dirs]) return pathlist @@ -68,6 +87,10 @@ def create_parser(): recursive_parser.add_argument('-r', '--recursive', action='store_true', dest='recursive', default=False, help='look for yaml files recursively') + recursive_parser.add_argument('-x', '--exclude', dest='exclude', + action='append', default=[], + help='paths to exclude when using recursive' + ' search, uses standard globbing.') subparser = parser.add_subparsers(help='update, test or delete job', dest='command') @@ -95,7 +118,7 @@ def create_parser(): # subparser: delete - parser_delete = subparser.add_parser('delete') + parser_delete = subparser.add_parser('delete', parents=[recursive_parser]) parser_delete.add_argument('name', help='name of job', nargs='+') parser_delete.add_argument('-p', '--path', default=None, help='colon-separated list of paths to' @@ -233,10 +256,13 @@ def execute(options, config): do_recurse = (getattr(options, 'recursive', False) or config.getboolean('job_builder', 'recursive')) + excludes = [e for elist in options.exclude + for e in elist.split(os.pathsep)] or \ + config.get('job_builder', 'exclude').split(os.pathsep) paths = [] for path in options.path: if do_recurse and os.path.isdir(path): - paths.extend(recurse_path(path)) + paths.extend(recurse_path(path, excludes)) else: paths.append(path) options.path = paths diff --git a/tests/cmd/subcommands/test_test.py b/tests/cmd/subcommands/test_test.py index c59f5ae73..037c29539 100644 --- a/tests/cmd/subcommands/test_test.py +++ b/tests/cmd/subcommands/test_test.py @@ -7,40 +7,41 @@ import jenkins from jenkins_jobs import cmd from jenkins_jobs.errors import JenkinsJobsException -from tests.cmd.test_cmd import CmdTestsBase from tests.base import mock +from tests.cmd.test_cmd import CmdTestsBase +from tests.cmd.test_recurse_path import fake_os_walk os_walk_return_values = { '/jjb_projects': [ - ('/jjb_projects', ('dir1', 'dir2', 'dir3'), ()), - ('/jjb_projects/dir1', ('bar',), ()), - ('/jjb_projects/dir2', ('baz',), ()), - ('/jjb_projects/dir3', (), ()), - ('/jjb_projects/dir1/bar', (), ()), - ('/jjb_projects/dir2/baz', (), ()), + ('/jjb_projects', (['dir1', 'dir2', 'dir3'], ())), + ('/jjb_projects/dir1', (['bar'], ())), + ('/jjb_projects/dir2', (['baz'], ())), + ('/jjb_projects/dir3', ([], ())), + ('/jjb_projects/dir1/bar', ([], ())), + ('/jjb_projects/dir2/baz', ([], ())), ], '/jjb_templates': [ - ('/jjb_templates', ('dir1', 'dir2', 'dir3'), ()), - ('/jjb_templates/dir1', ('bar',), ()), - ('/jjb_templates/dir2', ('baz',), ()), - ('/jjb_templates/dir3', (), ()), - ('/jjb_templates/dir1/bar', (), ()), - ('/jjb_templates/dir2/baz', (), ()), + ('/jjb_templates', (['dir1', 'dir2', 'dir3'], ())), + ('/jjb_templates/dir1', (['bar'], ())), + ('/jjb_templates/dir2', (['baz'], ())), + ('/jjb_templates/dir3', ([], ())), + ('/jjb_templates/dir1/bar', ([], ())), + ('/jjb_templates/dir2/baz', ([], ())), ], '/jjb_macros': [ - ('/jjb_macros', ('dir1', 'dir2', 'dir3'), ()), - ('/jjb_macros/dir1', ('bar',), ()), - ('/jjb_macros/dir2', ('baz',), ()), - ('/jjb_macros/dir3', (), ()), - ('/jjb_macros/dir1/bar', (), ()), - ('/jjb_macros/dir2/baz', (), ()), + ('/jjb_macros', (['dir1', 'dir2', 'dir3'], ())), + ('/jjb_macros/dir1', (['bar'], ())), + ('/jjb_macros/dir2', (['baz'], ())), + ('/jjb_macros/dir3', ([], ())), + ('/jjb_macros/dir1/bar', ([], ())), + ('/jjb_macros/dir2/baz', ([], ())), ], } def os_walk_side_effects(path_name, topdown): - return os_walk_return_values[path_name] + return fake_os_walk(os_walk_return_values[path_name])(path_name, topdown) @mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock) @@ -114,8 +115,7 @@ class TestTests(CmdTestsBase): path_list = os_walk_return_values.keys() paths = [] for path in path_list: - paths.extend([p for p, _, _ in - os_walk_return_values[path]]) + paths.extend([p for p, _ in os_walk_return_values[path]]) multipath = os.pathsep.join(path_list) @@ -132,6 +132,34 @@ class TestTests(CmdTestsBase): update_job_mock.assert_called_with(paths, [], output=args.output_dir) + @mock.patch('jenkins_jobs.cmd.Builder.update_job') + @mock.patch('jenkins_jobs.cmd.os.path.isdir') + @mock.patch('jenkins_jobs.cmd.os.walk') + def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock, + update_job_mock): + """ + Run test mode and pass multiple paths with recursive path option. + """ + + os_walk_mock.side_effect = os_walk_side_effects + isdir_mock.return_value = True + + path_list = os_walk_return_values.keys() + paths = [] + for path in path_list: + paths.extend([p for p, __ in os_walk_return_values[path] + if 'dir1' not in p and 'dir2' not in p]) + + multipath = os.pathsep.join(path_list) + + args = self.parser.parse_args(['test', '-r', multipath, '-x', + 'dir1:dir2']) + args.output_dir = mock.MagicMock() + + cmd.execute(args, self.config) + + update_job_mock.assert_called_with(paths, [], output=args.output_dir) + def test_console_output(self): """ Run test mode and verify that resulting XML gets sent to the console. diff --git a/tests/cmd/test_recurse_path.py b/tests/cmd/test_recurse_path.py new file mode 100644 index 000000000..42e24cfa3 --- /dev/null +++ b/tests/cmd/test_recurse_path.py @@ -0,0 +1,136 @@ +import os + +import mock +import testtools + +from jenkins_jobs import cmd + + +def fake_os_walk(paths): + """Helper function for mocking os.walk() where must test that manipulation + of the returned dirs variable works as expected + """ + paths_dict = dict(paths) + + def os_walk(top, topdown=True): + dirs, nondirs = paths_dict[top] + yield top, dirs, nondirs + for name in dirs: + # hard code use of '/' to ensure the test data can be defined as + # simple strings otherwise tests using this helper will break on + # platforms where os.path.sep is different. + new_path = "/".join([top, name]) + for x in os_walk(new_path, topdown): + yield x + return os_walk + + +# Testing the cmd module can sometimes result in the CacheStorage class +# attempting to create the cache directory multiple times as the tests +# are run in parallel. Stub out the CacheStorage to ensure that each +# test can safely create the object without effect. +@mock.patch('jenkins_jobs.builder.CacheStorage', mock.MagicMock) +class CmdRecursePath(testtools.TestCase): + + @mock.patch('jenkins_jobs.cmd.os.walk') + def test_recursive_path_option_exclude_pattern(self, oswalk_mock): + """ + Test paths returned by the recursive processing when using pattern + excludes. + + testing paths + /jjb_configs/dir1/test1/ + /jjb_configs/dir1/file + /jjb_configs/dir2/test2/ + /jjb_configs/dir3/bar/ + /jjb_configs/test3/bar/ + /jjb_configs/test3/baz/ + """ + + os_walk_paths = [ + ('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())), + ('/jjb_configs/dir1', (['test1'], ('file'))), + ('/jjb_configs/dir2', (['test2'], ())), + ('/jjb_configs/dir3', (['bar'], ())), + ('/jjb_configs/dir3/bar', ([], ())), + ('/jjb_configs/test3/bar', None), + ('/jjb_configs/test3/baz', None) + ] + + paths = [k for k, v in os_walk_paths if v is not None] + + oswalk_mock.side_effect = fake_os_walk(os_walk_paths) + self.assertEqual(paths, cmd.recurse_path('/jjb_configs', ['test*'])) + + @mock.patch('jenkins_jobs.cmd.os.walk') + def test_recursive_path_option_exclude_absolute(self, oswalk_mock): + """ + Test paths returned by the recursive processing when using absolute + excludes. + + testing paths + /jjb_configs/dir1/test1/ + /jjb_configs/dir1/file + /jjb_configs/dir2/test2/ + /jjb_configs/dir3/bar/ + /jjb_configs/test3/bar/ + /jjb_configs/test3/baz/ + """ + + os_walk_paths = [ + ('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())), + ('/jjb_configs/dir1', None), + ('/jjb_configs/dir2', (['test2'], ())), + ('/jjb_configs/dir3', (['bar'], ())), + ('/jjb_configs/test3', (['bar', 'baz'], ())), + ('/jjb_configs/dir2/test2', ([], ())), + ('/jjb_configs/dir3/bar', ([], ())), + ('/jjb_configs/test3/bar', ([], ())), + ('/jjb_configs/test3/baz', ([], ())) + ] + + paths = [k for k, v in os_walk_paths if v is not None] + + oswalk_mock.side_effect = fake_os_walk(os_walk_paths) + + self.assertEqual(paths, cmd.recurse_path('/jjb_configs', + ['/jjb_configs/dir1'])) + + @mock.patch('jenkins_jobs.cmd.os.walk') + def test_recursive_path_option_exclude_relative(self, oswalk_mock): + """ + Test paths returned by the recursive processing when using relative + excludes. + + testing paths + ./jjb_configs/dir1/test/ + ./jjb_configs/dir1/file + ./jjb_configs/dir2/test/ + ./jjb_configs/dir3/bar/ + ./jjb_configs/test3/bar/ + ./jjb_configs/test3/baz/ + """ + + os_walk_paths = [ + ('jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())), + ('jjb_configs/dir1', (['test'], ('file'))), + ('jjb_configs/dir2', (['test2'], ())), + ('jjb_configs/dir3', (['bar'], ())), + ('jjb_configs/test3', (['bar', 'baz'], ())), + ('jjb_configs/dir1/test', ([], ())), + ('jjb_configs/dir2/test2', ([], ())), + ('jjb_configs/dir3/bar', ([], ())), + ('jjb_configs/test3/bar', None), + ('jjb_configs/test3/baz', ([], ())) + ] + + rel_os_walk_paths = [ + (os.path.abspath( + os.path.join(os.path.curdir, k)), v) for k, v in os_walk_paths] + + paths = [k for k, v in rel_os_walk_paths if v is not None] + + oswalk_mock.side_effect = fake_os_walk(rel_os_walk_paths) + + self.assertEqual(paths, cmd.recurse_path('jjb_configs', + ['jjb_configs/test3/bar']))