From 2fc361949cea4135a5b6de35269d82720e495ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Vot=C3=A9?= Date: Wed, 4 May 2016 18:02:11 +0200 Subject: [PATCH] Add Pipeline job type and deprecate Workflow Since introduction of the Jenkins Pipeline plugin (https://wiki.jenkins-ci.org/display/JENKINS/Pipeline+Plugin), one can configure a Pipeline job using a Groovy file lying with the source code (see: https://jenkins.io/solutions/pipeline/). We add support here for this kind of jobs, introducing a new component: pipeline-scm. This component contains information about the repository to clone and the name of the file where the job configuration can be found. Since Workflow plugin is just an old name for the Pipeline plugin, Workflow job type can be deprecated in favor of Pipeline job type to avoid confusion and keep JJB in line with the terms used in Jenkins. Change-Id: I098b9673c44b1281fa0954193d8fadf273685386 Story: 2000573 Task: 2974 --- doc/source/project_pipeline.rst | 7 ++ jenkins_jobs/modules/general.py | 2 +- jenkins_jobs/modules/project_pipeline.py | 104 ++++++++++++++++++ jenkins_jobs/modules/project_workflow.py | 8 ++ jenkins_jobs/modules/scm.py | 24 ++++ setup.cfg | 2 + .../fixtures/project_pipeline_template001.xml | 25 +++++ .../project_pipeline_template001.yaml | 11 ++ .../fixtures/project_pipeline_template002.xml | 25 +++++ .../project_pipeline_template002.yaml | 22 ++++ .../fixtures/project_pipeline_template003.xml | 25 +++++ .../project_pipeline_template003.yaml | 23 ++++ .../fixtures/project_pipeline_template004.xml | 26 +++++ .../project_pipeline_template004.yaml | 11 ++ .../fixtures/project_pipeline_template005.xml | 65 +++++++++++ .../project_pipeline_template005.yaml | 41 +++++++ 16 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 doc/source/project_pipeline.rst create mode 100644 jenkins_jobs/modules/project_pipeline.py create mode 100644 tests/yamlparser/fixtures/project_pipeline_template001.xml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template001.yaml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template002.xml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template002.yaml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template003.xml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template003.yaml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template004.xml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template004.yaml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template005.xml create mode 100644 tests/yamlparser/fixtures/project_pipeline_template005.yaml diff --git a/doc/source/project_pipeline.rst b/doc/source/project_pipeline.rst new file mode 100644 index 000000000..67b0bbcdb --- /dev/null +++ b/doc/source/project_pipeline.rst @@ -0,0 +1,7 @@ +.. _project_pipeline: + +Pipeline Project +================ + +.. automodule:: project_pipeline + :members: diff --git a/jenkins_jobs/modules/general.py b/jenkins_jobs/modules/general.py index c4e67de9f..658d3c14e 100644 --- a/jenkins_jobs/modules/general.py +++ b/jenkins_jobs/modules/general.py @@ -22,7 +22,7 @@ Example: :Job Parameters: * **project-type**: Defaults to "freestyle", but "maven" as well as "multijob", "flow", - "workflow" or "externaljob" can also be specified. + "pipeline" or "externaljob" can also be specified. * **defaults**: Specifies a set of :ref:`defaults` to use for this job, defaults to diff --git a/jenkins_jobs/modules/project_pipeline.py b/jenkins_jobs/modules/project_pipeline.py new file mode 100644 index 000000000..ebf27fb8a --- /dev/null +++ b/jenkins_jobs/modules/project_pipeline.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2016 Mirantis, Inc. +# +# Based on jenkins_jobs/modules/project_workflow.py by +# Copyright (C) 2015 David Caro +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +The Pipeline Project module handles creating Jenkins Pipeline projects +(formerly known as the Workflow projects). +You may specify ``pipeline`` in the ``project-type`` attribute of +the :ref:`Job` definition. + +Requires the Jenkins :jenkins-wiki:`Pipeline Plugin `: + +In order to write an inline script within a job-template you have to escape the +curly braces by doubling them in the DSL: { -> {{ , otherwise it will be +interpreted by the python str.format() command. + +:Job Parameters: + * **sandbox** (`bool`): If the script should run in a sandbox (default + false) + * **dsl** (`str`): The DSL content or, + * **pipeline-scm** (`str`): in case "Pipeline as code" feature is used. + Then you should specify: + + * **scm**: single ``scm`` component (or a reference) describing the + source code repository + * **script-path**: path to the Groovy file containing the job's steps + (optional, default: ``Jenkinsfile``) + +Note that ``dsl`` and ``pipeline-scm`` parameters are mutually exclusive. + +Inline DSL job example: + + .. literalinclude:: + /../../tests/yamlparser/fixtures/project_pipeline_template001.yaml + +Inline DSL job template example: + + .. literalinclude:: + /../../tests/yamlparser/fixtures/project_pipeline_template002.yaml + +"Pipeline as code" example: + + .. literalinclude:: + /../../tests/yamlparser/fixtures/project_pipeline_template004.yaml + +"Pipeline as code" example using templates: + + .. literalinclude:: + /../../tests/yamlparser/fixtures/project_pipeline_template005.yaml + +.. _Pipeline as code: https://jenkins.io/solutions/pipeline/ + +""" +import xml.etree.ElementTree as XML + +from jenkins_jobs.errors import JenkinsJobsException +import jenkins_jobs.modules.base + + +class Pipeline(jenkins_jobs.modules.base.Base): + sequence = 0 + error_msg = ("You cannot declare both 'dsl' and 'pipeline-scm' on a " + "pipeline job") + + def root_xml(self, data): + xml_parent = XML.Element('flow-definition', + {'plugin': 'workflow-job'}) + if 'dsl' in data and 'pipeline-scm' in data: + raise JenkinsJobsException(self.error_msg) + if 'dsl' in data: + xml_definition = XML.SubElement(xml_parent, 'definition', + {'plugin': 'workflow-cps', + 'class': 'org.jenkinsci.plugins.' + 'workflow.cps.CpsFlowDefinition'}) + XML.SubElement(xml_definition, 'script').text = data['dsl'] + elif 'pipeline-scm' in data: + xml_definition = XML.SubElement(xml_parent, 'definition', { + 'plugin': 'workflow-cps', + 'class': 'org.jenkinsci.plugins.workflow.cps.' + 'CpsScmFlowDefinition'}) + else: + raise JenkinsJobsException("Either 'dsl' or 'pipeline-scm' " + "is required for pipeline job") + + needs_workspace = data.get('sandbox', False) + XML.SubElement(xml_definition, 'sandbox').text = str( + needs_workspace).lower() + + return xml_parent diff --git a/jenkins_jobs/modules/project_workflow.py b/jenkins_jobs/modules/project_workflow.py index ac4bb002a..4219033a6 100644 --- a/jenkins_jobs/modules/project_workflow.py +++ b/jenkins_jobs/modules/project_workflow.py @@ -18,6 +18,8 @@ """ +Deprecated: please use :ref:`project_pipeline` instead. + The workflow Project module handles creating Jenkins workflow projects. You may specify ``workflow`` in the ``project-type`` attribute of the :ref:`Job` definition. @@ -45,6 +47,7 @@ Job template example: /../../tests/yamlparser/fixtures/project_workflow_template002.yaml """ +import logging import xml.etree.ElementTree as XML from jenkins_jobs.errors import MissingAttributeError @@ -55,6 +58,11 @@ class Workflow(jenkins_jobs.modules.base.Base): sequence = 0 def root_xml(self, data): + logger = logging.getLogger(__name__) + logger.warning( + "Workflow job type is deprecated, please use Pipeline job type" + ) + xml_parent = XML.Element('flow-definition', {'plugin': 'workflow-job'}) xml_definition = XML.SubElement(xml_parent, 'definition', diff --git a/jenkins_jobs/modules/scm.py b/jenkins_jobs/modules/scm.py index 2b7331ada..86c4e815d 100644 --- a/jenkins_jobs/modules/scm.py +++ b/jenkins_jobs/modules/scm.py @@ -1284,3 +1284,27 @@ class SCM(jenkins_jobs.modules.base.Base): pass xml_parent.append(scms_parent) + + +class PipelineSCM(jenkins_jobs.modules.base.Base): + sequence = 30 + + component_type = 'pipeline-scm' + component_list_type = 'pipeline-scm' + + def gen_xml(self, xml_parent, data): + definition_parent = xml_parent.find('definition') + pipeline_dict = data.get(self.component_type, {}) + scms = pipeline_dict.get('scm') + if scms: + scms_count = len(scms) + if scms_count == 0: + raise JenkinsJobsException("'scm' missing or empty") + elif scms_count == 1: + self.registry.dispatch('scm', definition_parent, scms[0]) + XML.SubElement(definition_parent, 'scriptPath' + ).text = pipeline_dict.get('script-path', + 'Jenkinsfile') + else: + raise JenkinsJobsException('Only one SCM can be specified ' + 'as pipeline-scm') diff --git a/setup.cfg b/setup.cfg index fc240abcc..e12f5e9dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ jenkins_jobs.projects = matrix=jenkins_jobs.modules.project_matrix:Matrix maven=jenkins_jobs.modules.project_maven:Maven multijob=jenkins_jobs.modules.project_multijob:MultiJob + pipeline=jenkins_jobs.modules.project_pipeline:Pipeline workflow=jenkins_jobs.modules.project_workflow:Workflow jenkins_jobs.views = list=jenkins_jobs.modules.view_list:List @@ -76,6 +77,7 @@ jenkins_jobs.modules = metadata=jenkins_jobs.modules.metadata:Metadata notifications=jenkins_jobs.modules.notifications:Notifications parameters=jenkins_jobs.modules.parameters:Parameters + pipeline-scm=jenkins_jobs.modules.scm:PipelineSCM properties=jenkins_jobs.modules.properties:Properties publishers=jenkins_jobs.modules.publishers:Publishers reporters=jenkins_jobs.modules.reporters:Reporters diff --git a/tests/yamlparser/fixtures/project_pipeline_template001.xml b/tests/yamlparser/fixtures/project_pipeline_template001.xml new file mode 100644 index 000000000..f82dbe506 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template001.xml @@ -0,0 +1,25 @@ + + + + + false + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + diff --git a/tests/yamlparser/fixtures/project_pipeline_template001.yaml b/tests/yamlparser/fixtures/project_pipeline_template001.yaml new file mode 100644 index 000000000..2044da3bf --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template001.yaml @@ -0,0 +1,11 @@ +- job: + name: test_job + project-type: pipeline + dsl: | + build job: "job1" + parallel [ + 2a: build job: "job2a", + 2b: node "dummynode" { + sh "echo I'm alive!" + } + ] diff --git a/tests/yamlparser/fixtures/project_pipeline_template002.xml b/tests/yamlparser/fixtures/project_pipeline_template002.xml new file mode 100644 index 000000000..f92286df8 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template002.xml @@ -0,0 +1,25 @@ + + + + + false + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + diff --git a/tests/yamlparser/fixtures/project_pipeline_template002.yaml b/tests/yamlparser/fixtures/project_pipeline_template002.yaml new file mode 100644 index 000000000..bd0181f62 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template002.yaml @@ -0,0 +1,22 @@ +- job-template: + name: '{name}-unit-tests' + project-type: pipeline + dsl: | + build job: "job1" + parallel [ + 2a: build job: "job2a", + 2b: node "dummynode" {{ + sh "echo {isay}" + }} + ] + +- job-group: + name: '{name}-tests' + jobs: + - '{name}-unit-tests': + isay: 'hello' + +- project: + name: project-name + jobs: + - '{name}-tests' diff --git a/tests/yamlparser/fixtures/project_pipeline_template003.xml b/tests/yamlparser/fixtures/project_pipeline_template003.xml new file mode 100644 index 000000000..8274d6a0f --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template003.xml @@ -0,0 +1,25 @@ + + + + + true + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + diff --git a/tests/yamlparser/fixtures/project_pipeline_template003.yaml b/tests/yamlparser/fixtures/project_pipeline_template003.yaml new file mode 100644 index 000000000..a8fd61180 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template003.yaml @@ -0,0 +1,23 @@ +- job-template: + name: '{name}-unit-tests' + project-type: pipeline + dsl: | + build job: "job1" + parallel [ + 2a: build job: "job2a", + 2b: node "dummynode" {{ + sh "echo {isay}" + }} + ] + sandbox: true + +- job-group: + name: '{name}-tests' + jobs: + - '{name}-unit-tests': + isay: 'hello' + +- project: + name: project-name + jobs: + - '{name}-tests' diff --git a/tests/yamlparser/fixtures/project_pipeline_template004.xml b/tests/yamlparser/fixtures/project_pipeline_template004.xml new file mode 100644 index 000000000..942de03a0 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template004.xml @@ -0,0 +1,26 @@ + + + + true + + http://hg.example.org/test_job + BRANCH + default + true + + false + + Jenkinsfile.groovy + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + diff --git a/tests/yamlparser/fixtures/project_pipeline_template004.yaml b/tests/yamlparser/fixtures/project_pipeline_template004.yaml new file mode 100644 index 000000000..5f2f8f598 --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template004.yaml @@ -0,0 +1,11 @@ +- job: + name: test-job + project-type: pipeline + sandbox: true + pipeline-scm: + scm: + - hg: + url: http://hg.example.org/test_job + clean: true + script-path: Jenkinsfile.groovy + diff --git a/tests/yamlparser/fixtures/project_pipeline_template005.xml b/tests/yamlparser/fixtures/project_pipeline_template005.xml new file mode 100644 index 000000000..7bafcbfed --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template005.xml @@ -0,0 +1,65 @@ + + + + false + + http://hg.example.org/project + BRANCH + default + true + + false + + Jenkinsfile + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + qa@example.org + false + false + + + + + + + + + true + + http://hg.example.org/project + BRANCH + default + true + + false + + Jenkinsfile + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + dev@example.org + false + false + + + + diff --git a/tests/yamlparser/fixtures/project_pipeline_template005.yaml b/tests/yamlparser/fixtures/project_pipeline_template005.yaml new file mode 100644 index 000000000..851feaf9d --- /dev/null +++ b/tests/yamlparser/fixtures/project_pipeline_template005.yaml @@ -0,0 +1,41 @@ +- scm: + name: project-scm + scm: + - hg: + url: http://hg.example.org/project + clean: true + +- job-template: + name: '{name}-unit-tests' + project-type: pipeline + pipeline-scm: + scm: + - project-scm + sandbox: true + publishers: + - email: + recipients: '{mail-to}' + +- job-template: + name: '{name}-perf-tests' + project-type: pipeline + pipeline-scm: + scm: + - project-scm + sandbox: false + publishers: + - email: + recipients: '{mail-to}' + +- job-group: + name: '{name}-tests' + jobs: + - '{name}-unit-tests': + mail-to: dev@example.org + - '{name}-perf-tests': + mail-to: qa@example.org + +- project: + name: project-name + jobs: + - '{name}-tests'