diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 507d5fa1a..f419cfed0 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -74,6 +74,12 @@ job_builder section (Optional) If set to True, jenkins job builder will search for job definition files recursively +**allow_duplicates** + (Optional) By default `jenkins-jobs` will abort any time a duplicate macro, + template, job-group or job name is encountered as it cannot establish the + correct one to use. When this option is set to True, only a warning is + emitted. + jenkins section ^^^^^^^^^^^^^^^ diff --git a/etc/jenkins_jobs.ini-sample b/etc/jenkins_jobs.ini-sample index 90030757a..3753c861c 100644 --- a/etc/jenkins_jobs.ini-sample +++ b/etc/jenkins_jobs.ini-sample @@ -3,6 +3,7 @@ ignore_cache=True keep_descriptions=False include_path=.:scripts:~/git/ recursive=False +allow_duplicates=False [jenkins] user=jenkins diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index 472e5cd09..66215a67a 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -157,6 +157,9 @@ class YamlParser(object): "named '{0}'. Missing indent?" .format(n)) name = dfn['name'] + if name in group: + self._handle_dups("Duplicate entry found: '{0}' is " + "already defined".format(name)) group[name] = dfn self.data[cls] = group @@ -164,6 +167,15 @@ class YamlParser(object): with open(fn) as fp: self.parse_fp(fp) + def _handle_dups(self, message): + + if not (self.config and self.config.has_section('job_builder') and + self.config.getboolean('job_builder', 'allow_duplicates')): + logger.error(message) + raise JenkinsJobsException(message) + else: + logger.warn(message) + def getJob(self, name): job = self.data.get('job', {}).get(name, None) if not job: @@ -208,6 +220,8 @@ class YamlParser(object): self.getXMLForJob(job) for project in self.data.get('project', {}).values(): logger.debug("XMLifying project '{0}'".format(project['name'])) + # use a set to check for duplicate job references in projects + seen = set() for jobspec in project.get('jobs', []): if isinstance(jobspec, dict): # Singleton dict containing dict of job-specific params @@ -220,6 +234,11 @@ class YamlParser(object): job = self.getJob(jobname) if job: # Just naming an existing defined job + if jobname in seen: + self._handle_dups("Duplicate job '{0}' specified " + "for project '{1}'".format( + jobname, project['name'])) + seen.add(jobname) continue # see if it's a job group group = self.getJobGroup(jobname) @@ -235,6 +254,12 @@ class YamlParser(object): group_jobparams = {} job = self.getJob(group_jobname) if job: + if group_jobname in seen: + self._handle_dups( + "Duplicate job '{0}' specified for " + "project '{1}'".format(group_jobname, + project['name'])) + seen.add(group_jobname) continue template = self.getJobTemplate(group_jobname) # Allow a group to override parameters set by a project @@ -259,6 +284,15 @@ class YamlParser(object): raise JenkinsJobsException("Failed to find suitable " "template named '{0}'" .format(jobname)) + # check for duplicate generated jobs + seen = set() + # walk the list in reverse so that last definition wins + for job in self.jobs[::-1]: + if job.name in seen: + self._handle_dups("Duplicate definitions for job '{0}' " + "specified".format(job.name)) + self.jobs.remove(job) + seen.add(job.name) def getXMLForTemplateJob(self, project, template, jobs_filter=None): dimensions = [] diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index 1fd3cebbb..a1cb3696d 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -31,6 +31,7 @@ DEFAULT_CONF = """ keep_descriptions=False ignore_cache=False recursive=False +allow_duplicates=False [jenkins] url=http://localhost:8080/ diff --git a/tests/duplicates/__init__.py b/tests/duplicates/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/allow_duplicates001.conf b/tests/duplicates/fixtures/allow_duplicates001.conf new file mode 100644 index 000000000..81f12f1df --- /dev/null +++ b/tests/duplicates/fixtures/allow_duplicates001.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_duplicates = True diff --git a/tests/duplicates/fixtures/allow_duplicates001.xml b/tests/duplicates/fixtures/allow_duplicates001.xml new file mode 100644 index 000000000..e059d06ce --- /dev/null +++ b/tests/duplicates/fixtures/allow_duplicates001.xml @@ -0,0 +1,49 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + ** + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/allow_duplicates001.yaml b/tests/duplicates/fixtures/allow_duplicates001.yaml new file mode 100644 index 000000000..1d2223742 --- /dev/null +++ b/tests/duplicates/fixtures/allow_duplicates001.yaml @@ -0,0 +1,18 @@ +- project: + name: duplicates + version: + - 1.1 + jobs: + - 'duplicates002_{version}' + +- job-template: + name: 'duplicates002_{version}' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + +- job: + name: duplicates002_1.1 + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/fixtures/allow_job_group001.conf b/tests/duplicates/fixtures/allow_job_group001.conf new file mode 100644 index 000000000..81f12f1df --- /dev/null +++ b/tests/duplicates/fixtures/allow_job_group001.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_duplicates = True diff --git a/tests/duplicates/fixtures/allow_job_group001.xml b/tests/duplicates/fixtures/allow_job_group001.xml new file mode 100644 index 000000000..519736942 --- /dev/null +++ b/tests/duplicates/fixtures/allow_job_group001.xml @@ -0,0 +1,19 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + step2 + + + + + diff --git a/tests/duplicates/fixtures/allow_job_group001.yaml b/tests/duplicates/fixtures/allow_job_group001.yaml new file mode 100644 index 000000000..d82da476a --- /dev/null +++ b/tests/duplicates/fixtures/allow_job_group001.yaml @@ -0,0 +1,24 @@ +- job-template: + name: '{name}-1' + builders: + - shell: step1 + +- job-template: + name: '{name}-2' + builders: + - shell: step2 + +- job-group: + name: 'group-1' + jobs: + - '{name}-1' + +- job-group: + name: 'group-1' + jobs: + - '{name}-2' + +- project: + name: project-name + jobs: + - 'group-1' diff --git a/tests/duplicates/fixtures/allow_macros001.conf b/tests/duplicates/fixtures/allow_macros001.conf new file mode 100644 index 000000000..81f12f1df --- /dev/null +++ b/tests/duplicates/fixtures/allow_macros001.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_duplicates = True diff --git a/tests/duplicates/fixtures/allow_macros001.xml b/tests/duplicates/fixtures/allow_macros001.xml new file mode 100644 index 000000000..c24d57210 --- /dev/null +++ b/tests/duplicates/fixtures/allow_macros001.xml @@ -0,0 +1,49 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/second.git + + + + + origin/stable-2 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/allow_macros001.yaml b/tests/duplicates/fixtures/allow_macros001.yaml new file mode 100644 index 000000000..63eb29db8 --- /dev/null +++ b/tests/duplicates/fixtures/allow_macros001.yaml @@ -0,0 +1,21 @@ +- scm: + name: project-scm + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/first.git + branches: + - origin/stable-1 + +- scm: + name: project-scm + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/second.git + branches: + - origin/stable-2 + +- job: + name: duplicate_macros + scm: + - project-scm + diff --git a/tests/duplicates/fixtures/allow_projects001.conf b/tests/duplicates/fixtures/allow_projects001.conf new file mode 100644 index 000000000..81f12f1df --- /dev/null +++ b/tests/duplicates/fixtures/allow_projects001.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_duplicates = True diff --git a/tests/duplicates/fixtures/allow_projects001.xml b/tests/duplicates/fixtures/allow_projects001.xml new file mode 100644 index 000000000..0b4b1a1b3 --- /dev/null +++ b/tests/duplicates/fixtures/allow_projects001.xml @@ -0,0 +1,49 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git + + + + + origin/stable-2 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/allow_projects001.yaml b/tests/duplicates/fixtures/allow_projects001.yaml new file mode 100644 index 000000000..aa1f8dc30 --- /dev/null +++ b/tests/duplicates/fixtures/allow_projects001.yaml @@ -0,0 +1,15 @@ +- project: + name: duplicate-templates + jobs: + - '{name}-001': + version: 1 + - '{name}-001': + version: 2 + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git + branches: + - origin/stable-{version} diff --git a/tests/duplicates/fixtures/allow_templates001.conf b/tests/duplicates/fixtures/allow_templates001.conf new file mode 100644 index 000000000..81f12f1df --- /dev/null +++ b/tests/duplicates/fixtures/allow_templates001.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_duplicates = True diff --git a/tests/duplicates/fixtures/allow_templates001.xml b/tests/duplicates/fixtures/allow_templates001.xml new file mode 100644 index 000000000..0b4b1a1b3 --- /dev/null +++ b/tests/duplicates/fixtures/allow_templates001.xml @@ -0,0 +1,49 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git + + + + + origin/stable-2 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/allow_templates001.yaml b/tests/duplicates/fixtures/allow_templates001.yaml new file mode 100644 index 000000000..3ce5dab92 --- /dev/null +++ b/tests/duplicates/fixtures/allow_templates001.yaml @@ -0,0 +1,20 @@ +- project: + name: duplicate-templates + jobs: + - '{name}-001' + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + branches: + - origin/stable-1 + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git + branches: + - origin/stable-2 diff --git a/tests/duplicates/fixtures/duplicates001.xml b/tests/duplicates/fixtures/duplicates001.xml new file mode 100644 index 000000000..6f2e535ab --- /dev/null +++ b/tests/duplicates/fixtures/duplicates001.xml @@ -0,0 +1,98 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + stable/1.1 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + stable/2.0 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/duplicates001.yaml b/tests/duplicates/fixtures/duplicates001.yaml new file mode 100644 index 000000000..674a52cd4 --- /dev/null +++ b/tests/duplicates/fixtures/duplicates001.yaml @@ -0,0 +1,16 @@ +- project: + name: duplicates + version: + - 1.1 + jobs: + - 'duplicates002_{version}' + - 'duplicates002_{version}': + version: 2.0 + +- job-template: + name: 'duplicates002_{version}' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + branches: + - 'stable/{version}' diff --git a/tests/duplicates/fixtures/duplicates002.xml b/tests/duplicates/fixtures/duplicates002.xml new file mode 100644 index 000000000..4c285c547 --- /dev/null +++ b/tests/duplicates/fixtures/duplicates002.xml @@ -0,0 +1,98 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + origin/stable-1.1 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + + + + + origin/stable-2.0 + + + + + false + false + false + false + false + true + false + false + Default + + + + + + false + + false + false + + + + + diff --git a/tests/duplicates/fixtures/duplicates002.yaml b/tests/duplicates/fixtures/duplicates002.yaml new file mode 100644 index 000000000..48a7aa32f --- /dev/null +++ b/tests/duplicates/fixtures/duplicates002.yaml @@ -0,0 +1,21 @@ +- project: + name: duplicates + version: + - 1.1 + jobs: + - 'duplicate_job_group' + - 'duplicate_job_group': + version: 2.0 + +- job-group: + name: duplicate_job_group + jobs: + - 'duplicates002_{version}' + +- job-template: + name: 'duplicates002_{version}' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + branches: + - 'origin/stable-{version}' diff --git a/tests/duplicates/fixtures/exception_duplicates001.xml b/tests/duplicates/fixtures/exception_duplicates001.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_duplicates001.yaml b/tests/duplicates/fixtures/exception_duplicates001.yaml new file mode 100644 index 000000000..223722c9c --- /dev/null +++ b/tests/duplicates/fixtures/exception_duplicates001.yaml @@ -0,0 +1,11 @@ +- job: + name: duplicate001 + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + +- job: + name: duplicate001 + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/fixtures/exception_duplicates002.xml b/tests/duplicates/fixtures/exception_duplicates002.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_duplicates002.yaml b/tests/duplicates/fixtures/exception_duplicates002.yaml new file mode 100644 index 000000000..1d2223742 --- /dev/null +++ b/tests/duplicates/fixtures/exception_duplicates002.yaml @@ -0,0 +1,18 @@ +- project: + name: duplicates + version: + - 1.1 + jobs: + - 'duplicates002_{version}' + +- job-template: + name: 'duplicates002_{version}' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + +- job: + name: duplicates002_1.1 + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/fixtures/exception_job_group001.xml b/tests/duplicates/fixtures/exception_job_group001.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_job_group001.yaml b/tests/duplicates/fixtures/exception_job_group001.yaml new file mode 100644 index 000000000..928043833 --- /dev/null +++ b/tests/duplicates/fixtures/exception_job_group001.yaml @@ -0,0 +1,20 @@ +- job-template: + name: '{name}-1' + builders: + - shell: step1 + +- job-template: + name: '{name}-2' + builders: + - shell: step2 + +- job-group: + name: 'group-1' + +- job-group: + name: 'group-1' + +- project: + name: project-name + jobs: + - 'group-1' diff --git a/tests/duplicates/fixtures/exception_macros001.xml b/tests/duplicates/fixtures/exception_macros001.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_macros001.yaml b/tests/duplicates/fixtures/exception_macros001.yaml new file mode 100644 index 000000000..d97a64a6b --- /dev/null +++ b/tests/duplicates/fixtures/exception_macros001.yaml @@ -0,0 +1,21 @@ +- scm: + name: project-scm + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/first.git + branches: + - origin/master + +- scm: + name: project-scm + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/second.git + branches: + - origin/master + +- job: + name: duplicate_macros + scm: + - project-scm + diff --git a/tests/duplicates/fixtures/exception_projects001.xml b/tests/duplicates/fixtures/exception_projects001.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_projects001.yaml b/tests/duplicates/fixtures/exception_projects001.yaml new file mode 100644 index 000000000..38c2d708e --- /dev/null +++ b/tests/duplicates/fixtures/exception_projects001.yaml @@ -0,0 +1,13 @@ +- project: + name: duplicates + version: + - 1.1 + jobs: + - 'duplicates002_1.1' + - 'duplicates002_1.1' + +- job: + name: duplicates002_1.1 + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/fixtures/exception_projects002.xml b/tests/duplicates/fixtures/exception_projects002.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_projects002.yaml b/tests/duplicates/fixtures/exception_projects002.yaml new file mode 100644 index 000000000..b6d53ad86 --- /dev/null +++ b/tests/duplicates/fixtures/exception_projects002.yaml @@ -0,0 +1,13 @@ +- project: + name: duplicate-groups + jobs: + - 'group-001' + - 'group-001' + +- job-group: + name: 'group-001' + jobs: + - dummy-job + +- job: + name: dummy-job diff --git a/tests/duplicates/fixtures/exception_projects003.xml b/tests/duplicates/fixtures/exception_projects003.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_projects003.yaml b/tests/duplicates/fixtures/exception_projects003.yaml new file mode 100644 index 000000000..0672de6cf --- /dev/null +++ b/tests/duplicates/fixtures/exception_projects003.yaml @@ -0,0 +1,11 @@ +- project: + name: duplicate-templates + jobs: + - '{name}-001' + - '{name}-001' + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/fixtures/exception_templates001.xml b/tests/duplicates/fixtures/exception_templates001.xml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/duplicates/fixtures/exception_templates001.yaml b/tests/duplicates/fixtures/exception_templates001.yaml new file mode 100644 index 000000000..d33c1db5d --- /dev/null +++ b/tests/duplicates/fixtures/exception_templates001.yaml @@ -0,0 +1,16 @@ +- project: + name: duplicate-templates + jobs: + - '{name}-001' + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git + +- job-template: + name: '{name}-001' + scm: + - git: + url: ssh://jenkins@review.openstack.org:29418/openstack-infra/git-review.git diff --git a/tests/duplicates/test_duplicates.py b/tests/duplicates/test_duplicates.py new file mode 100644 index 000000000..1f84ad387 --- /dev/null +++ b/tests/duplicates/test_duplicates.py @@ -0,0 +1,36 @@ +# Joint copyright: +# - Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import mock +from testtools import TestCase, ExpectedException +from testscenarios.testcase import TestWithScenarios +from tests.base import get_scenarios, SingleJobTestCase +from jenkins_jobs.errors import JenkinsJobsException + + +class TestCaseModuleDuplicates(TestWithScenarios, TestCase, + SingleJobTestCase): + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + scenarios = get_scenarios(fixtures_path) + + @mock.patch('jenkins_jobs.builder.logger', autospec=True) + def test_yaml_snippet(self, mock_logger): + + if self.in_filename.startswith("exception_"): + with ExpectedException(JenkinsJobsException, "^Duplicate .*"): + super(TestCaseModuleDuplicates, self).test_yaml_snippet() + else: + super(TestCaseModuleDuplicates, self).test_yaml_snippet()