Do not output XML if job name not specified ('pipeline', 'trigger-builds')

Done in order to enable specifying build pipelines at the project level.
Pipelines are built using the 'pipeline' plugin or the 'trigger-builds'
plugin.  The downstream job to be executed is specified at the project
level, and substituted into the template.  Ignoring definitions where
the downstream job is blank allows the pipeline to be easily terminated.

Also adding a samples/ directory, which currently contains an example of
pipeline creation, together with a new tools/run-compare-xml-samples.sh
to run the before/after test on the samples.

Change-Id: Icc324d1485e22db7824ad784890db48eb3e48f8f
Reviewed-on: https://review.openstack.org/13706
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins

doc/source/configuration.rst

@@ -45,6 +45,8 @@ later.  There are a few basic optional fields for a Job definition::
   should not use the ``global`` defaults, use this field to specify a
   different set of defaults.
 
+.. _job-template:
+
 Job Template
 ^^^^^^^^^^^^
 
@@ -72,6 +74,8 @@ define a template that you can use to create jobs with a `Project`_
 definition.  It's name will depend on what is supplied to the
 `Project`_.
 
+.. _project:
+
 Project
 ^^^^^^^
 

jenkins_jobs/modules/builders.py

@@ -39,6 +39,9 @@ Example::
 
 import xml.etree.ElementTree as XML
 import jenkins_jobs.modules.base
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def shell(parser, xml_parent, data):
@@ -158,6 +161,9 @@ def trigger_builds(parser, xml_parent, data):
                    'hudson.plugins.parameterizedtrigger.TriggerBuilder')
     configs = XML.SubElement(tbuilder, 'configs')
     for project_def in data:
+        if 'project' not in project_def or project_def['project'] == '':
+            logger.debug("No project specified - skipping trigger-build")
+            continue
         tconfig = XML.SubElement(configs,
             'hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig')
         tconfigs = XML.SubElement(tconfig, 'configs')
@@ -181,6 +187,10 @@ def trigger_builds(parser, xml_parent, data):
         build_all_nodes_with_label = XML.SubElement(tconfig,
                                                     'buildAllNodesWithLabel')
         build_all_nodes_with_label.text = 'false'
+    # If configs is empty, remove the entire tbuilder tree.
+    if(len(configs) == 0):
+        logger.debug("Pruning empty TriggerBuilder tree.")
+        xml_parent.remove(tbuilder)
 
 
 def builders_from(parser, xml_parent, data):

jenkins_jobs/modules/publishers.py

@@ -578,11 +578,20 @@ def pipeline(parser, xml_parent, data):
 
       publishers:
         - pipleline: deploy
+
+    You can build pipeline jobs that are re-usable in different pipelines by
+    using a :ref:`job-template` to define the pipeline jobs,
+    and variable substitution to specify the name of
+    the downstream job in the pipeline.
+    Job-specific substitutions are useful here (see :ref:`project`).
+
+    See 'samples/pipeline.yaml' for an example pipeline implementation.
     """
-    pippub = XML.SubElement(xml_parent,
+    if data != '':
+        pippub = XML.SubElement(xml_parent,
                             'au.com.centrumsystems.hudson.plugin.'
                             'buildpipeline.trigger.BuildPipelineTrigger')
-    XML.SubElement(pippub, 'downstreamProjectNames').text = data
+        XML.SubElement(pippub, 'downstreamProjectNames').text = data
 
 
 def email(parser, xml_parent, data):

samples/README.md

@@ -0,0 +1,8 @@
+# Jenkins Job Builder Examples #
+
+These examples demonstrate the use of particular features of Jenkins
+Job Builder that can be used as starting points for new projects.
+
+These examples are also used in the test framework - if you add a new
+example here, please ensure that it can be processed by Jenkins Job
+Builder.

samples/pipeline.yaml

@@ -0,0 +1,73 @@
+# Sample YAML representation of a flexible pipeline for use with the
+# Jenkins Build Pipeline Plugin
+# (https://wiki.jenkins-ci.org/display/JENKINS/Build+Pipeline+Plugin)
+# The pipeline is defined at the project level, using job-specific variable
+# substitution to specify the next job in the pipeline ('pipeline-next') 
+# for each job.
+#
+# This example defines the skeleton of two deployment pipelines: one for
+# a 'base_os' subsystem to three chef servers, and the second to
+# a 'database' subsystem to two chef servers.
+# 
+# The pipeline uses an automatically-executed job in '{subsys}-prepare'
+# (using the Parameterized Trigger Plugin - 
+# https://wiki.jenkins-ci.org/display/JENKINS/Parameterized+Trigger+Plugin),
+# and manually-triggered jobs using the Pipeline publisher (e.g.
+# '{subsys}-initialise').
+#
+# Note how each project defines a different pipeline, with a different
+# length, but re-uses the same job templates.
+
+- project:
+    name: deploy_base_os
+    subsys: base_os
+    jobs:
+      - '{subsys}-prepare':
+          pipeline-next: '{subsys}-initialise'
+      - '{subsys}-initialise':
+          pipeline-next: '{subsys}-upload-chef-server1'
+      - '{subsys}-upload-chef-server1':
+          pipeline-next: '{subsys}-upload-chef-server2'
+      - '{subsys}-upload-chef-server2':
+          pipeline-next: '{subsys}-upload-chef-server3'
+      - '{subsys}-upload-chef-server3':
+          pipeline-next: ''
+
+- project:
+    name: deploy_database
+    subsys: db
+    jobs:
+      - '{subsys}-prepare':
+          pipeline-next: '{subsys}-initialise'
+      - '{subsys}-initialise':
+          pipeline-next: '{subsys}-upload-chef-server1'
+      - '{subsys}-upload-chef-server1':
+          pipeline-next: '{subsys}-upload-chef-server2'
+      - '{subsys}-upload-chef-server2':
+          pipeline-next: ''
+
+- job-template:
+    name: '{subsys}-prepare'
+    builders:
+      - trigger-builds:
+          - project: '{pipeline-next}'
+
+- job-template:
+    name: '{subsys}-initialise'
+    publishers:
+      - pipeline: '{pipeline-next}'
+        
+- job-template:
+    name: '{subsys}-upload-chef-server1'
+    publishers:
+      - pipeline: '{pipeline-next}'
+
+- job-template:
+    name: '{subsys}-upload-chef-server2'
+    publishers:
+      - pipeline: '{pipeline-next}'
+
+- job-template:
+    name: '{subsys}-upload-chef-server3'
+    publishers:
+      - pipeline: '{pipeline-next}'

tools/run-compare-xml-samples.sh

@@ -0,0 +1,62 @@
+#!/bin/bash -e
+
+# Copied and modified from tools/run-compare-xml.sh
+
+# Copyright (c) 2012, AT&T Labs, Yun Mao <yunmao@gmail.com>
+# All Rights Reserved.
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Assuming that all samples can be sensibly run at the same time.
+
+rm -fr .test
+mkdir -p .test/old/config
+mkdir -p .test/old/out
+mkdir -p .test/new/config
+mkdir -p .test/new/out
+cp samples/*.yaml .test/old/config
+cp .test/old/config/* .test/new/config
+
+GITHEAD=`git rev-parse HEAD`
+
+# First generate output from HEAD~1
+git checkout HEAD~1
+tox -e compare-xml-old
+
+# Then use that as a reference to compare against HEAD
+git checkout $GITHEAD
+tox -e compare-xml-new
+
+CHANGED=0
+for x in `(cd .test/old/out && find -type f)`
+do
+    if ! diff -u .test/old/out/$x .test/new/out/$x >/dev/null 2>&1
+    then
+	CHANGED=1
+	echo "============================================================"
+	echo $x
+	echo "------------------------------------------------------------"
+    fi
+    diff -u .test/old/out/$x .test/new/out/$x || /bin/true
+done
+
+echo
+echo "You are in detached HEAD mode. If you are a developer"
+echo "and not very familiar with git, you might want to do"
+echo "'git checkout branch-name' to go back to your branch."
+
+if [ "$CHANGED" -eq "1" ]; then
+    exit 1
+fi
+exit 0