jjb/jenkins-job-builder
Code Issues Proposed changes

Support excludes when recursively processing paths

Browse Source 
Include an additional option to exclude some paths/patterns from being
included in the search to allow a complex hierarchy of directories to be
referenced under a single path. Provide support to allow the user to
specify patterns, relative and/or absolute path references.

Implements support for excluding based on absolute path, relative path
and based on simple shell globing patterns.

Change-Id: I236b364c268fd2bf5511a2c6d9a1c87914f3b086
This commit is contained in:
Darragh Bailey 2014-04-07 23:46:35 +01:00
parent 74c8ac4561
commit c5c0eb4f16
5 changed files with 277 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
doc/source/installation.rst
View File

@ -74,6 +74,14 @@ job_builder section
(Optional) If set to True, jenkins job builder will search for job
definition files recursively
**exclude**
(Optional) If set to a list of values separated by ':', these paths will be
excluded from the list of paths to be processed when searching recursively.
Values containing no ``/`` will be matched against directory names at all
levels, those starting with ``/`` will be considered absolute, while others
containing a ``/`` somewhere other than the start of the value will be
considered relative to the starting path.
**allow_duplicates**
(Optional) By default `jenkins-jobs` will abort any time a duplicate macro,
template, job-group or job name is encountered as it cannot establish the
@ -155,6 +163,59 @@ Jenkins instances suited to various needs you may want to share configuration
between those instances (global). Furthermore, there may be various ways you
would like to structure jobs within a given instance.
Recursive Searching of Paths
----------------------------
In addition to passing multiple paths to JJB it is also possible to enable
recursive searching to process all yaml files in the tree beneath each path.
For example::
For a tree:
/path/
to/
defs/
ci_jobs/
release_jobs/
globals/
macros/
templates/
jenkins-jobs update -r /path/to/defs:/path/to/globals
JJB will search defs/ci_jobs, defs/release_jobs, globals/macros and
globals/templates in addition to the defs and globals trees.
Excluding Paths
---------------
To allow a complex tree of jobs where some jobs are managed differently without
needing to explicitly provide each path, the recursive path processing supports
excluding paths based on absolute paths, relative paths and patterns. For
example::
For a tree:
/path/
to/
defs/
ci_jobs/
manual/
release_jobs/
manual/
qa_jobs/
globals/
macros/
templates/
special/
jenkins-jobs update -r -x man*:./qa_jobs -x /path/to/defs/globals/special \
/path/to/defs:/path/to/globals
JJB search the given paths, ignoring the directories qa_jobs, ci_jobs/manual,
release_jobs/manual, and globals/special when building the list of yaml files
to be processed. Absolute paths are denoted by starting from the root,
relative by containing the path separator, and patterns by having neither.
Patterns use simple shell globing to match directories.
Deleting Jobs
^^^^^^^^^^^^^
Jenkins Job Builder supports deleting jobs from Jenkins.

1
etc/jenkins_jobs.ini-sample
View File

@ -3,6 +3,7 @@ ignore_cache=True
keep_descriptions=False
include_path=.:scripts:~/git/
recursive=False
exclude=.*:manual:./development
allow_duplicates=False
[jenkins]

32
jenkins_jobs/cmd.py
View File

@ -15,6 +15,7 @@
import argparse
from six.moves import configparser, StringIO
import fnmatch
import logging
import os
import platform
@ -33,6 +34,7 @@ DEFAULT_CONF = """
keep_descriptions=False
ignore_cache=False
recursive=False
exclude=.*
allow_duplicates=False
[jenkins]
@ -51,11 +53,28 @@ def confirm(question):
sys.exit('Aborted')
def recurse_path(root):
def recurse_path(root, excludes=None):
if excludes is None:
excludes = []
basepath = os.path.realpath(root)
pathlist = [basepath]
patterns = [e for e in excludes if os.path.sep not in e]
absolute = [e for e in excludes if os.path.isabs(e)]
relative = [e for e in excludes if os.path.sep in e and
not os.path.isabs(e)]
for root, dirs, files in os.walk(basepath, topdown=True):
dirs[:] = [
d for d in dirs
if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])
if not any([fnmatch.fnmatch(os.path.abspath(os.path.join(root, d)),
path)
for path in absolute])
if not any([fnmatch.fnmatch(os.path.relpath(os.path.join(root, d)),
path)
for path in relative])
]
pathlist.extend([os.path.join(root, path) for path in dirs])
return pathlist
@ -68,6 +87,10 @@ def create_parser():
recursive_parser.add_argument('-r', '--recursive', action='store_true',
dest='recursive', default=False,
help='look for yaml files recursively')
recursive_parser.add_argument('-x', '--exclude', dest='exclude',
action='append', default=[],
help='paths to exclude when using recursive'
' search, uses standard globbing.')
subparser = parser.add_subparsers(help='update, test or delete job',
dest='command')
@ -95,7 +118,7 @@ def create_parser():
# subparser: delete
parser_delete = subparser.add_parser('delete')
parser_delete = subparser.add_parser('delete', parents=[recursive_parser])
parser_delete.add_argument('name', help='name of job', nargs='+')
parser_delete.add_argument('-p', '--path', default=None,
help='colon-separated list of paths to'
@ -233,10 +256,13 @@ def execute(options, config):
do_recurse = (getattr(options, 'recursive', False) or
config.getboolean('job_builder', 'recursive'))
excludes = [e for elist in options.exclude
for e in elist.split(os.pathsep)] or \
config.get('job_builder', 'exclude').split(os.pathsep)
paths = []
for path in options.path:
if do_recurse and os.path.isdir(path):
paths.extend(recurse_path(path))
paths.extend(recurse_path(path, excludes))
else:
paths.append(path)
options.path = paths

72
tests/cmd/subcommands/test_test.py
View File

@ -7,40 +7,41 @@ import jenkins
from jenkins_jobs import cmd
from jenkins_jobs.errors import JenkinsJobsException
from tests.cmd.test_cmd import CmdTestsBase
from tests.base import mock
from tests.cmd.test_cmd import CmdTestsBase
from tests.cmd.test_recurse_path import fake_os_walk
os_walk_return_values = {
'/jjb_projects': [
('/jjb_projects', ('dir1', 'dir2', 'dir3'), ()),
('/jjb_projects/dir1', ('bar',), ()),
('/jjb_projects/dir2', ('baz',), ()),
('/jjb_projects/dir3', (), ()),
('/jjb_projects/dir1/bar', (), ()),
('/jjb_projects/dir2/baz', (), ()),
('/jjb_projects', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_projects/dir1', (['bar'], ())),
('/jjb_projects/dir2', (['baz'], ())),
('/jjb_projects/dir3', ([], ())),
('/jjb_projects/dir1/bar', ([], ())),
('/jjb_projects/dir2/baz', ([], ())),
],
'/jjb_templates': [
('/jjb_templates', ('dir1', 'dir2', 'dir3'), ()),
('/jjb_templates/dir1', ('bar',), ()),
('/jjb_templates/dir2', ('baz',), ()),
('/jjb_templates/dir3', (), ()),
('/jjb_templates/dir1/bar', (), ()),
('/jjb_templates/dir2/baz', (), ()),
('/jjb_templates', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_templates/dir1', (['bar'], ())),
('/jjb_templates/dir2', (['baz'], ())),
('/jjb_templates/dir3', ([], ())),
('/jjb_templates/dir1/bar', ([], ())),
('/jjb_templates/dir2/baz', ([], ())),
],
'/jjb_macros': [
('/jjb_macros', ('dir1', 'dir2', 'dir3'), ()),
('/jjb_macros/dir1', ('bar',), ()),
('/jjb_macros/dir2', ('baz',), ()),
('/jjb_macros/dir3', (), ()),
('/jjb_macros/dir1/bar', (), ()),
('/jjb_macros/dir2/baz', (), ()),
('/jjb_macros', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_macros/dir1', (['bar'], ())),
('/jjb_macros/dir2', (['baz'], ())),
('/jjb_macros/dir3', ([], ())),
('/jjb_macros/dir1/bar', ([], ())),
('/jjb_macros/dir2/baz', ([], ())),
],
}
def os_walk_side_effects(path_name, topdown):
return os_walk_return_values[path_name]
return fake_os_walk(os_walk_return_values[path_name])(path_name, topdown)
@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
@ -114,8 +115,7 @@ class TestTests(CmdTestsBase):
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([p for p, _, _ in
os_walk_return_values[path]])
paths.extend([p for p, _ in os_walk_return_values[path]])
multipath = os.pathsep.join(path_list)
@ -132,6 +132,34 @@ class TestTests(CmdTestsBase):
update_job_mock.assert_called_with(paths, [], output=args.output_dir)
@mock.patch('jenkins_jobs.cmd.Builder.update_job')
@mock.patch('jenkins_jobs.cmd.os.path.isdir')
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock,
update_job_mock):
"""
Run test mode and pass multiple paths with recursive path option.
"""
os_walk_mock.side_effect = os_walk_side_effects
isdir_mock.return_value = True
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([p for p, __ in os_walk_return_values[path]
if 'dir1' not in p and 'dir2' not in p])
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', '-r', multipath, '-x',
'dir1:dir2'])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
update_job_mock.assert_called_with(paths, [], output=args.output_dir)
def test_console_output(self):
"""
Run test mode and verify that resulting XML gets sent to the console.

136
tests/cmd/test_recurse_path.py Normal file
View File

@ -0,0 +1,136 @@
import os
import mock
import testtools
from jenkins_jobs import cmd
def fake_os_walk(paths):
"""Helper function for mocking os.walk() where must test that manipulation
of the returned dirs variable works as expected
"""
paths_dict = dict(paths)
def os_walk(top, topdown=True):
dirs, nondirs = paths_dict[top]
yield top, dirs, nondirs
for name in dirs:
# hard code use of '/' to ensure the test data can be defined as
# simple strings otherwise tests using this helper will break on
# platforms where os.path.sep is different.
new_path = "/".join([top, name])
for x in os_walk(new_path, topdown):
yield x
return os_walk
# Testing the cmd module can sometimes result in the CacheStorage class
# attempting to create the cache directory multiple times as the tests
# are run in parallel. Stub out the CacheStorage to ensure that each
# test can safely create the object without effect.
@mock.patch('jenkins_jobs.builder.CacheStorage', mock.MagicMock)
class CmdRecursePath(testtools.TestCase):
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_path_option_exclude_pattern(self, oswalk_mock):
"""
Test paths returned by the recursive processing when using pattern
excludes.
testing paths
/jjb_configs/dir1/test1/
/jjb_configs/dir1/file
/jjb_configs/dir2/test2/
/jjb_configs/dir3/bar/
/jjb_configs/test3/bar/
/jjb_configs/test3/baz/
"""
os_walk_paths = [
('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
('/jjb_configs/dir1', (['test1'], ('file'))),
('/jjb_configs/dir2', (['test2'], ())),
('/jjb_configs/dir3', (['bar'], ())),
('/jjb_configs/dir3/bar', ([], ())),
('/jjb_configs/test3/bar', None),
('/jjb_configs/test3/baz', None)
]
paths = [k for k, v in os_walk_paths if v is not None]
oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
self.assertEqual(paths, cmd.recurse_path('/jjb_configs', ['test*']))
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_path_option_exclude_absolute(self, oswalk_mock):
"""
Test paths returned by the recursive processing when using absolute
excludes.
testing paths
/jjb_configs/dir1/test1/
/jjb_configs/dir1/file
/jjb_configs/dir2/test2/
/jjb_configs/dir3/bar/
/jjb_configs/test3/bar/
/jjb_configs/test3/baz/
"""
os_walk_paths = [
('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
('/jjb_configs/dir1', None),
('/jjb_configs/dir2', (['test2'], ())),
('/jjb_configs/dir3', (['bar'], ())),
('/jjb_configs/test3', (['bar', 'baz'], ())),
('/jjb_configs/dir2/test2', ([], ())),
('/jjb_configs/dir3/bar', ([], ())),
('/jjb_configs/test3/bar', ([], ())),
('/jjb_configs/test3/baz', ([], ()))
]
paths = [k for k, v in os_walk_paths if v is not None]
oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
self.assertEqual(paths, cmd.recurse_path('/jjb_configs',
['/jjb_configs/dir1']))
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_path_option_exclude_relative(self, oswalk_mock):
"""
Test paths returned by the recursive processing when using relative
excludes.
testing paths
./jjb_configs/dir1/test/
./jjb_configs/dir1/file
./jjb_configs/dir2/test/
./jjb_configs/dir3/bar/
./jjb_configs/test3/bar/
./jjb_configs/test3/baz/
"""
os_walk_paths = [
('jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
('jjb_configs/dir1', (['test'], ('file'))),
('jjb_configs/dir2', (['test2'], ())),
('jjb_configs/dir3', (['bar'], ())),
('jjb_configs/test3', (['bar', 'baz'], ())),
('jjb_configs/dir1/test', ([], ())),
('jjb_configs/dir2/test2', ([], ())),
('jjb_configs/dir3/bar', ([], ())),
('jjb_configs/test3/bar', None),
('jjb_configs/test3/baz', ([], ()))
]
rel_os_walk_paths = [
(os.path.abspath(
os.path.join(os.path.curdir, k)), v) for k, v in os_walk_paths]
paths = [k for k, v in rel_os_walk_paths if v is not None]
oswalk_mock.side_effect = fake_os_walk(rel_os_walk_paths)
self.assertEqual(paths, cmd.recurse_path('jjb_configs',
['jjb_configs/test3/bar']))