jjb/jenkins-job-builder
Code Issues Proposed changes

Separate out custom yaml tags into separate classes

Browse Source 
Implement custom tags for including files by subclassing YAMLObject to
allow PyYAML to easily convert from the Yaml nodes.

Change to support both lists and strings under the same tag name and
deprecate the old tag naming. Rename tests using the deprecated tags to
ensure they are not used for examples. Remove old tests that simply
duplicate others.

Change-Id: I1a8d3376ea6e4918a2adb05fb856b939a3124d74
This commit is contained in:
Darragh Bailey 2015-05-06 01:09:45 +01:00
parent 59a8342f85
commit 3d728c71a8
23 changed files with 313 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
jenkins_jobs/local_yaml.py
View File

@ -23,7 +23,7 @@ managed separately to the yaml job configurations. A specific usage of this is
inlining scripts contained in separate files, although such tags may also be
used to simplify usage of macros or job templates.
The tag ``!include`` will treat the following string as file which should be
The tag ``!include:`` will treat the following string as file which should be
parsed as yaml configuration data.
Example:
@ -35,65 +35,63 @@ Example:
.. literalinclude:: /../../tests/yamlparser/fixtures/include001.yaml.inc
The tag ``!include-raw`` will treat the following file as a data blob, which
should be read into the calling yaml construct without any further parsing.
Any data in a file included through this tag, will be treated as string data.
The tag ``!include-raw:`` will treat the given string or list of strings as
filenames to be opened as one or more data blob, which should be read into
the calling yaml construct without any further parsing. Any data in a file
included through this tag, will be treated as string data.
Example:
Examples:
.. literalinclude:: /../../tests/localyaml/fixtures/include-raw001.yaml
contents of include-raw001-hello-world.sh:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
contents of include-raw001-vars.sh:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-vars.sh
using a list of files:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-vars.sh
/../../tests/localyaml/fixtures/include-raw-multi001.yaml
The tag ``!include-raw-escape:`` treats the given string or list of strings as
filenames to be opened as one or more data blobs, which should be escaped
before being read in as string data. This allows job-templates to use this tag
to include scripts from files without needing to escape braces in the original
file.
The tag ``!include-raw-escape`` treats the given file as a data blob, which
should be escaped before being read in as string data. This allows
job-templates to use this tag to include scripts from files without
needing to escape braces in the original file.
Example:
Examples:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw-escaped001.yaml
contents of include-raw001-hello-world.sh:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
contents of include-raw001-vars.sh:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-vars.sh
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw001-vars.sh
Variants for the raw include tags ``!include-raw:`` and
``!include-raw-escape:`` accept a list of files. All of the specified files
are concatenated and included as string data.
Example:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw-multi001.yaml
using a list of files:
.. literalinclude::
/../../tests/localyaml/fixtures/include-raw-escaped-multi001.yaml
For all the multi file includes, the files are simply appended using a newline
character.
"""
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
import functools
import io
import logging
@ -101,6 +99,12 @@ import re
import os
import yaml
from yaml.constructor import BaseConstructor
from yaml import YAMLObject
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
logger = logging.getLogger(__name__)
@ -167,11 +171,9 @@ class LocalAnchorLoader(yaml.Loader):
class LocalLoader(OrderedConstructor, LocalAnchorLoader):
"""Subclass for yaml.Loader which handles the local tags 'include',
'include-raw' and 'include-raw-escaped' to specify a file to include data
from and whether to parse it as additional yaml, treat it as a data blob
or additionally escape the data contained. These are specified in yaml
files by "!include path/to/file.yaml".
"""Subclass for yaml.Loader which handles storing the search_path and
escape_callback functions for use by the custom YAML objects to find files
and escape the content where required.
Constructor access a list of search paths to look under for the given
file following each tag, taking the first match found. Search path by
@ -209,19 +211,12 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
self.search_path.append(os.path.normpath(p))
if 'escape_callback' in kwargs:
self._escape = kwargs.pop('escape_callback')
self.escape_callback = kwargs.pop('escape_callback')
else:
self.escape_callback = self._escape
super(LocalLoader, self).__init__(*args, **kwargs)
# Add tag constructors
self.add_constructor('!include', self._include_tag)
self.add_constructor('!include-raw', self._include_raw_tag)
self.add_constructor('!include-raw-escape',
self._include_raw_escape_tag)
self.add_constructor('!include-raw:', self._include_raw_tag_multi)
self.add_constructor('!include-raw-escape:',
self._include_raw_escape_tag_multi)
# constructor to preserve order of maps and ensure that the order of
# keys returned is consistent across multiple python versions
self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
@ -232,8 +227,21 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
os.path.dirname(self.stream.name)))
self.search_path.append(os.path.normpath(os.path.curdir))
def _find_file(self, filename):
for dirname in self.search_path:
def _escape(self, data):
return re.sub(r'({|})', r'\1\1', data)
class BaseYAMLObject(YAMLObject):
yaml_loader = LocalLoader
yaml_dumper = yaml.Dumper
class YamlInclude(BaseYAMLObject):
yaml_tag = u'!include:'
@classmethod
def _find_file(cls, filename, search_path):
for dirname in search_path:
candidate = os.path.expanduser(os.path.join(dirname, filename))
if os.path.isfile(candidate):
logger.info("Including file '{0}' from path '{1}'"
@ -241,43 +249,76 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
return candidate
return filename
def _include_tag(self, loader, node):
filename = self._find_file(loader.construct_yaml_str(node))
with io.open(filename, 'r', encoding='utf-8') as f:
data = yaml.load(f, functools.partial(LocalLoader,
search_path=self.search_path
))
return data
def _include_raw_tag(self, loader, node):
filename = self._find_file(loader.construct_yaml_str(node))
@classmethod
def _open_file(cls, loader, scalar_node):
filename = cls._find_file(loader.construct_yaml_str(scalar_node),
loader.search_path)
try:
with io.open(filename, 'r', encoding='utf-8') as f:
data = f.read()
return f.read()
except:
logger.error("Failed to include file using search path: '{0}'"
.format(':'.join(self.search_path)))
.format(':'.join(loader.search_path)))
raise
@classmethod
def _from_file(cls, loader, node):
data = yaml.load(cls._open_file(loader, node),
functools.partial(cls.yaml_loader,
search_path=loader.search_path))
return data
def _include_raw_tag_multi(self, loader, node):
if not isinstance(node, yaml.SequenceNode):
@classmethod
def from_yaml(cls, loader, node):
if isinstance(node, yaml.ScalarNode):
return cls._from_file(loader, node)
elif isinstance(node, yaml.SequenceNode):
return u'\n'.join(cls._from_file(loader, scalar_node)
for scalar_node in node.value)
else:
raise yaml.constructor.ConstructorError(
None, None,
"expected a sequence node, but found %s" % node.id,
node.start_mark)
None, None, "expected either a sequence or scalar node, but "
"found %s" % node.id, node.start_mark)
return '\n'.join(self._include_raw_tag(loader, scalar_node)
for scalar_node in node.value)
def _include_raw_escape_tag(self, loader, node):
return self._escape(self._include_raw_tag(loader, node))
class YamlIncludeRaw(YamlInclude):
yaml_tag = u'!include-raw:'
def _include_raw_escape_tag_multi(self, loader, node):
return self._escape(self._include_raw_tag_multi(loader, node))
@classmethod
def _from_file(cls, loader, node):
return cls._open_file(loader, node)
def _escape(self, data):
return re.sub(r'({|})', r'\1\1', data)
class YamlIncludeRawEscape(YamlIncludeRaw):
yaml_tag = u'!include-raw-escape:'
@classmethod
def from_yaml(cls, loader, node):
return loader.escape_callback(YamlIncludeRaw.from_yaml(loader, node))
class DeprecatedTag(BaseYAMLObject):
@classmethod
def from_yaml(cls, loader, node):
logger.warn("tag '%s' is deprecated, switch to using '%s'",
cls.yaml_tag, cls._new.yaml_tag)
return cls._new.from_yaml(loader, node)
class YamlIncludeDeprecated(DeprecatedTag):
yaml_tag = u'!include'
_new = YamlInclude
class YamlIncludeRawDeprecated(DeprecatedTag):
yaml_tag = u'!include-raw'
_new = YamlIncludeRaw
class YamlIncludeRawEscapeDeprecated(DeprecatedTag):
yaml_tag = u'!include-raw-escape'
_new = YamlIncludeRawEscape
def load(stream, **kwargs):

24
tests/localyaml/fixtures/deprecated-include-raw-escaped001.json Normal file
View File

@ -0,0 +1,24 @@
[
{
"job-template": {
"name": "test-job-include-raw-{num}",
"builders": [
{
"shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
},
{
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n\n"
}
]
}
},
{
"project": {
"name": "test-job-template-1",
"num": 1,
"jobs": [
"test-job-include-raw-{num}"
]
}
}
]

13
tests/localyaml/fixtures/deprecated-include-raw-escaped001.yaml Normal file
View File

@ -0,0 +1,13 @@
- job-template:
name: test-job-include-raw-{num}
builders:
- shell:
!include-raw-escape include-raw001-hello-world.sh
- shell:
!include-raw-escape include-raw001-vars.sh
- project:
name: test-job-template-1
num: 1
jobs:
- 'test-job-include-raw-{num}'

15
tests/localyaml/fixtures/deprecated-include-raw001.json Normal file
View File

@ -0,0 +1,15 @@
[
{
"job": {
"name": "test-job-include-raw-1",
"builders": [
{
"shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
},
{
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n\n"
}
]
}
}
]

7
tests/localyaml/fixtures/deprecated-include-raw001.yaml Normal file
View File

@ -0,0 +1,7 @@
- job:
name: test-job-include-raw-1
builders:
- shell:
!include-raw include-raw001-hello-world.sh
- shell:
!include-raw include-raw001-vars.sh

43
tests/localyaml/fixtures/deprecated-include001.json Normal file
View File

@ -0,0 +1,43 @@
[
{
"job": {
"name": "test-job-1",
"builders": [
{
"copyartifact": {
"project": "foo",
"filter": "*.tar.gz",
"target": "/home/foo",
"which-build": "last-successful",
"optional": true,
"flatten": true,
"parameter-filters": "PUBLISH=true"
}
},
{
"copyartifact": {
"project": "bar",
"filter": "*.tar.gz",
"target": "/home/foo",
"which-build": "specific-build",
"optional": true,
"flatten": true,
"parameter-filters": "PUBLISH=true",
"build-number": 123
}
},
{
"copyartifact": {
"project": "baz",
"filter": "*.tar.gz",
"target": "/home/foo",
"which-build": "upstream-build",
"optional": true,
"flatten": true,
"parameter-filters": "PUBLISH=true"
}
}
]
}
}
]

4
tests/localyaml/fixtures/deprecated-include001.yaml Normal file
View File

@ -0,0 +1,4 @@
- job:
name: test-job-1
builders:
!include include001.yaml.inc

2
tests/localyaml/fixtures/exception_include001.yaml
View File

@ -12,4 +12,4 @@
- job:
name: test-job-1
builders:
!include exception_include001.yaml.inc
!include: exception_include001.yaml.inc

4
tests/localyaml/fixtures/include-raw-escaped001.yaml
View File

@ -2,9 +2,9 @@
name: test-job-include-raw-{num}
builders:
- shell:
!include-raw-escape include-raw001-hello-world.sh
!include-raw-escape: include-raw001-hello-world.sh
- shell:
!include-raw-escape include-raw001-vars.sh
!include-raw-escape: include-raw001-vars.sh
- project:
name: test-job-template-1

4
tests/localyaml/fixtures/include-raw001.yaml
View File

@ -2,6 +2,6 @@
name: test-job-include-raw-1
builders:
- shell:
!include-raw include-raw001-hello-world.sh
!include-raw: include-raw001-hello-world.sh
- shell:
!include-raw include-raw001-vars.sh
!include-raw: include-raw001-vars.sh

2
tests/localyaml/fixtures/include001.yaml
View File

@ -1,4 +1,4 @@
- job:
name: test-job-1
builders:
!include include001.yaml.inc
!include: include001.yaml.inc

0
tests/yamlparser/fixtures/include002.xml → tests/yamlparser/fixtures/deprecated-include001.xml
View File

2
tests/yamlparser/fixtures/include-raw-escape001.yaml
View File

@ -11,7 +11,7 @@
keep-system-variables: true
builders:
- shell:
!include-raw-escape include-raw-escape001-echo-vars.sh
!include-raw-escape: include-raw-escape001-echo-vars.sh
- project:

58
tests/yamlparser/fixtures/include-raw001.xml
View File

@ -19,20 +19,48 @@
</EnvInjectJobProperty>
</properties>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>#!/bin/bash
#
# Sample script showing how the yaml include-raw tag can be used
# to inline scripts that are maintained outside of the jenkins
# job yaml configuration.
echo &quot;hello world&quot;
exit 0
</command>
</hudson.tasks.Shell>
</builders>
<builders/>
<publishers/>
<buildWrappers/>
<buildWrappers>
<hudson.plugins.build__timeout.BuildTimeoutWrapper>
<timeoutMinutes>3</timeoutMinutes>
<failBuild>true</failBuild>
<writingDescription>false</writingDescription>
<timeoutPercentage>150</timeoutPercentage>
<timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
<timeoutType>elastic</timeoutType>
</hudson.plugins.build__timeout.BuildTimeoutWrapper>
<org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
<buildSteps>
<hudson.tasks.Shell>
<command>#!/bin/bash
echo &quot;Doing somethiung cool&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>#!/bin/zsh
echo &quot;Doing somethin cool with zsh&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Ant>
<targets>target1 target2</targets>
<antName>Standard Ant</antName>
</hudson.tasks.Ant>
<EnvInjectBuilder>
<info>
<propertiesFilePath>example.prop</propertiesFilePath>
<propertiesContent>EXAMPLE=foo-bar</propertiesContent>
</info>
</EnvInjectBuilder>
</buildSteps>
</org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
<com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
<includes>file1,file2*.txt</includes>
<excludes>file2bad.txt</excludes>
<flatten>false</flatten>
<includeAntExcludes>false</includeAntExcludes>
<relativeTo>userContent</relativeTo>
<hudsonHomeRelative>false</hudsonHomeRelative>
</com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
</buildWrappers>
</project>

43
tests/yamlparser/fixtures/include-raw001.yaml
View File

@ -1,10 +1,45 @@
# vim: sw=4 ts=4 et
- wrapper:
name: timeout-wrapper
wrappers:
- timeout:
fail: true
elastic-percentage: 150
elastic-default-timeout: 90
type: elastic
- wrapper:
name: pre-scm-shell-ant
wrappers:
- pre-scm-buildstep:
- shell:
!include-raw: include-raw002-cool.sh
- shell:
!include-raw: include-raw002-cool.zsh
- ant:
targets: "target1 target2"
ant-name: "Standard Ant"
- inject:
properties-file: example.prop
properties-content: EXAMPLE=foo-bar
- wrapper:
name: copy-files
wrappers:
- copy-to-slave:
includes:
- file1
- file2*.txt
excludes:
- file2bad.txt
- job:
name: test-job-2
name: test-job-3
wrappers:
!include: include001.yaml.inc
properties:
- inject:
keep-build-variables: true
keep-system-variables: true
builders:
- shell:
!include-raw ../../localyaml/fixtures/include-raw001-hello-world.sh

66
tests/yamlparser/fixtures/include-raw002.xml
View File

@ -1,66 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties>
<EnvInjectJobProperty>
<info>
<loadFilesFromMaster>false</loadFilesFromMaster>
</info>
<on>true</on>
<keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
<keepBuildVariables>true</keepBuildVariables>
<overrideBuildParameters>false</overrideBuildParameters>
</EnvInjectJobProperty>
</properties>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers>
<hudson.plugins.build__timeout.BuildTimeoutWrapper>
<timeoutMinutes>3</timeoutMinutes>
<failBuild>true</failBuild>
<writingDescription>false</writingDescription>
<timeoutPercentage>150</timeoutPercentage>
<timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
<timeoutType>elastic</timeoutType>
</hudson.plugins.build__timeout.BuildTimeoutWrapper>
<org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
<buildSteps>
<hudson.tasks.Shell>
<command>#!/bin/bash
echo &quot;Doing somethiung cool&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>#!/bin/zsh
echo &quot;Doing somethin cool with zsh&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Ant>
<targets>target1 target2</targets>
<antName>Standard Ant</antName>
</hudson.tasks.Ant>
<EnvInjectBuilder>
<info>
<propertiesFilePath>example.prop</propertiesFilePath>
<propertiesContent>EXAMPLE=foo-bar</propertiesContent>
</info>
</EnvInjectBuilder>
</buildSteps>
</org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
<com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
<includes>file1,file2*.txt</includes>
<excludes>file2bad.txt</excludes>
<flatten>false</flatten>
<includeAntExcludes>false</includeAntExcludes>
<relativeTo>userContent</relativeTo>
<hudsonHomeRelative>false</hudsonHomeRelative>
</com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
</buildWrappers>
</project>

45
tests/yamlparser/fixtures/include-raw002.yaml
View File

@ -1,45 +0,0 @@
# vim: sw=4 ts=4 et
- wrapper:
name: timeout-wrapper
wrappers:
- timeout:
fail: true
elastic-percentage: 150
elastic-default-timeout: 90
type: elastic
- wrapper:
name: pre-scm-shell-ant
wrappers:
- pre-scm-buildstep:
- shell:
!include-raw include-raw002-cool.sh
- shell:
!include-raw include-raw002-cool.zsh
- ant:
targets: "target1 target2"
ant-name: "Standard Ant"
- inject:
properties-file: example.prop
properties-content: EXAMPLE=foo-bar
- wrapper:
name: copy-files
wrappers:
- copy-to-slave:
includes:
- file1
- file2*.txt
excludes:
- file2bad.txt
- job:
name: test-job-3
wrappers:
!include include001.yaml.inc
properties:
- inject:
keep-build-variables: true
keep-system-variables: true

2
tests/yamlparser/fixtures/include-rawunicode001.yaml
View File

@ -3,7 +3,7 @@
wrappers:
- pre-scm-buildstep:
- shell:
!include-raw include-rawunicode001-cool.sh
!include-raw: include-rawunicode001-cool.sh
- job:
name: test-unicode-raw-include-wrapper

2
tests/yamlparser/fixtures/include001.yaml
View File

@ -38,7 +38,7 @@
- job:
name: test-job-1
wrappers:
!include include001.yaml.inc
!include: include001.yaml.inc
properties:
- inject:
keep-build-variables: true

16
tests/yamlparser/fixtures/include002.yaml
View File

@ -1,16 +0,0 @@
# vim: sw=4 ts=4 et
- default:
name: default-timeout-wrapper
timeout: &timeout
fail: true
elastic-percentage: 150
elastic-default-timeout: 90
type: elastic
# include that uses timeout alias
- wrapper: !include include002_1.yaml.inc
- job:
name: test-job-1
wrappers:
!include include002.yaml.inc

1
tests/yamlparser/fixtures/include002.yaml.inc
View File

@ -1 +0,0 @@
- timeout-wrapper

3
tests/yamlparser/fixtures/include002_1.yaml.inc
View File

@ -1,3 +0,0 @@
name: timeout-wrapper
wrappers:
- timeout: *timeout

2
tests/yamlparser/fixtures/include_path002.yaml
View File

@ -5,4 +5,4 @@
- job-template:
name: 'include_path'
description: !include test.inc
description: !include: test.inc