diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index fae350ba1..97afa4723 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -23,7 +23,7 @@ managed separately to the yaml job configurations. A specific usage of this is inlining scripts contained in separate files, although such tags may also be used to simplify usage of macros or job templates. -The tag ``!include`` will treat the following string as file which should be +The tag ``!include:`` will treat the following string as file which should be parsed as yaml configuration data. Example: @@ -35,65 +35,63 @@ Example: .. literalinclude:: /../../tests/yamlparser/fixtures/include001.yaml.inc -The tag ``!include-raw`` will treat the following file as a data blob, which -should be read into the calling yaml construct without any further parsing. -Any data in a file included through this tag, will be treated as string data. +The tag ``!include-raw:`` will treat the given string or list of strings as +filenames to be opened as one or more data blob, which should be read into +the calling yaml construct without any further parsing. Any data in a file +included through this tag, will be treated as string data. -Example: +Examples: .. literalinclude:: /../../tests/localyaml/fixtures/include-raw001.yaml contents of include-raw001-hello-world.sh: - .. literalinclude:: - /../../tests/localyaml/fixtures/include-raw001-hello-world.sh + .. literalinclude:: + /../../tests/localyaml/fixtures/include-raw001-hello-world.sh contents of include-raw001-vars.sh: + .. literalinclude:: + /../../tests/localyaml/fixtures/include-raw001-vars.sh + + using a list of files: + .. literalinclude:: - /../../tests/localyaml/fixtures/include-raw001-vars.sh + /../../tests/localyaml/fixtures/include-raw-multi001.yaml + +The tag ``!include-raw-escape:`` treats the given string or list of strings as +filenames to be opened as one or more data blobs, which should be escaped +before being read in as string data. This allows job-templates to use this tag +to include scripts from files without needing to escape braces in the original +file. -The tag ``!include-raw-escape`` treats the given file as a data blob, which -should be escaped before being read in as string data. This allows -job-templates to use this tag to include scripts from files without -needing to escape braces in the original file. - - -Example: +Examples: .. literalinclude:: /../../tests/localyaml/fixtures/include-raw-escaped001.yaml contents of include-raw001-hello-world.sh: - .. literalinclude:: - /../../tests/localyaml/fixtures/include-raw001-hello-world.sh + .. literalinclude:: + /../../tests/localyaml/fixtures/include-raw001-hello-world.sh contents of include-raw001-vars.sh: - .. literalinclude:: - /../../tests/localyaml/fixtures/include-raw001-vars.sh + .. literalinclude:: + /../../tests/localyaml/fixtures/include-raw001-vars.sh - -Variants for the raw include tags ``!include-raw:`` and -``!include-raw-escape:`` accept a list of files. All of the specified files -are concatenated and included as string data. - -Example: - - .. literalinclude:: - /../../tests/localyaml/fixtures/include-raw-multi001.yaml + using a list of files: .. literalinclude:: /../../tests/localyaml/fixtures/include-raw-escaped-multi001.yaml + +For all the multi file includes, the files are simply appended using a newline +character. + """ -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict import functools import io import logging @@ -101,6 +99,12 @@ import re import os import yaml from yaml.constructor import BaseConstructor +from yaml import YAMLObject + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict logger = logging.getLogger(__name__) @@ -167,11 +171,9 @@ class LocalAnchorLoader(yaml.Loader): class LocalLoader(OrderedConstructor, LocalAnchorLoader): - """Subclass for yaml.Loader which handles the local tags 'include', - 'include-raw' and 'include-raw-escaped' to specify a file to include data - from and whether to parse it as additional yaml, treat it as a data blob - or additionally escape the data contained. These are specified in yaml - files by "!include path/to/file.yaml". + """Subclass for yaml.Loader which handles storing the search_path and + escape_callback functions for use by the custom YAML objects to find files + and escape the content where required. Constructor access a list of search paths to look under for the given file following each tag, taking the first match found. Search path by @@ -209,19 +211,12 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader): self.search_path.append(os.path.normpath(p)) if 'escape_callback' in kwargs: - self._escape = kwargs.pop('escape_callback') + self.escape_callback = kwargs.pop('escape_callback') + else: + self.escape_callback = self._escape super(LocalLoader, self).__init__(*args, **kwargs) - # Add tag constructors - self.add_constructor('!include', self._include_tag) - self.add_constructor('!include-raw', self._include_raw_tag) - self.add_constructor('!include-raw-escape', - self._include_raw_escape_tag) - self.add_constructor('!include-raw:', self._include_raw_tag_multi) - self.add_constructor('!include-raw-escape:', - self._include_raw_escape_tag_multi) - # constructor to preserve order of maps and ensure that the order of # keys returned is consistent across multiple python versions self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, @@ -232,8 +227,21 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader): os.path.dirname(self.stream.name))) self.search_path.append(os.path.normpath(os.path.curdir)) - def _find_file(self, filename): - for dirname in self.search_path: + def _escape(self, data): + return re.sub(r'({|})', r'\1\1', data) + + +class BaseYAMLObject(YAMLObject): + yaml_loader = LocalLoader + yaml_dumper = yaml.Dumper + + +class YamlInclude(BaseYAMLObject): + yaml_tag = u'!include:' + + @classmethod + def _find_file(cls, filename, search_path): + for dirname in search_path: candidate = os.path.expanduser(os.path.join(dirname, filename)) if os.path.isfile(candidate): logger.info("Including file '{0}' from path '{1}'" @@ -241,43 +249,76 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader): return candidate return filename - def _include_tag(self, loader, node): - filename = self._find_file(loader.construct_yaml_str(node)) - with io.open(filename, 'r', encoding='utf-8') as f: - data = yaml.load(f, functools.partial(LocalLoader, - search_path=self.search_path - )) - return data - - def _include_raw_tag(self, loader, node): - filename = self._find_file(loader.construct_yaml_str(node)) + @classmethod + def _open_file(cls, loader, scalar_node): + filename = cls._find_file(loader.construct_yaml_str(scalar_node), + loader.search_path) try: with io.open(filename, 'r', encoding='utf-8') as f: - data = f.read() + return f.read() except: logger.error("Failed to include file using search path: '{0}'" - .format(':'.join(self.search_path))) + .format(':'.join(loader.search_path))) raise + + @classmethod + def _from_file(cls, loader, node): + data = yaml.load(cls._open_file(loader, node), + functools.partial(cls.yaml_loader, + search_path=loader.search_path)) return data - def _include_raw_tag_multi(self, loader, node): - if not isinstance(node, yaml.SequenceNode): + @classmethod + def from_yaml(cls, loader, node): + if isinstance(node, yaml.ScalarNode): + return cls._from_file(loader, node) + elif isinstance(node, yaml.SequenceNode): + return u'\n'.join(cls._from_file(loader, scalar_node) + for scalar_node in node.value) + else: raise yaml.constructor.ConstructorError( - None, None, - "expected a sequence node, but found %s" % node.id, - node.start_mark) + None, None, "expected either a sequence or scalar node, but " + "found %s" % node.id, node.start_mark) - return '\n'.join(self._include_raw_tag(loader, scalar_node) - for scalar_node in node.value) - def _include_raw_escape_tag(self, loader, node): - return self._escape(self._include_raw_tag(loader, node)) +class YamlIncludeRaw(YamlInclude): + yaml_tag = u'!include-raw:' - def _include_raw_escape_tag_multi(self, loader, node): - return self._escape(self._include_raw_tag_multi(loader, node)) + @classmethod + def _from_file(cls, loader, node): + return cls._open_file(loader, node) - def _escape(self, data): - return re.sub(r'({|})', r'\1\1', data) + +class YamlIncludeRawEscape(YamlIncludeRaw): + yaml_tag = u'!include-raw-escape:' + + @classmethod + def from_yaml(cls, loader, node): + return loader.escape_callback(YamlIncludeRaw.from_yaml(loader, node)) + + +class DeprecatedTag(BaseYAMLObject): + + @classmethod + def from_yaml(cls, loader, node): + logger.warn("tag '%s' is deprecated, switch to using '%s'", + cls.yaml_tag, cls._new.yaml_tag) + return cls._new.from_yaml(loader, node) + + +class YamlIncludeDeprecated(DeprecatedTag): + yaml_tag = u'!include' + _new = YamlInclude + + +class YamlIncludeRawDeprecated(DeprecatedTag): + yaml_tag = u'!include-raw' + _new = YamlIncludeRaw + + +class YamlIncludeRawEscapeDeprecated(DeprecatedTag): + yaml_tag = u'!include-raw-escape' + _new = YamlIncludeRawEscape def load(stream, **kwargs): diff --git a/tests/localyaml/fixtures/deprecated-include-raw-escaped001.json b/tests/localyaml/fixtures/deprecated-include-raw-escaped001.json new file mode 100644 index 000000000..268b0aab7 --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include-raw-escaped001.json @@ -0,0 +1,24 @@ +[ + { + "job-template": { + "name": "test-job-include-raw-{num}", + "builders": [ + { + "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n" + }, + { + "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n\n" + } + ] + } + }, + { + "project": { + "name": "test-job-template-1", + "num": 1, + "jobs": [ + "test-job-include-raw-{num}" + ] + } + } +] diff --git a/tests/localyaml/fixtures/deprecated-include-raw-escaped001.yaml b/tests/localyaml/fixtures/deprecated-include-raw-escaped001.yaml new file mode 100644 index 000000000..2853d03ba --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include-raw-escaped001.yaml @@ -0,0 +1,13 @@ +- job-template: + name: test-job-include-raw-{num} + builders: + - shell: + !include-raw-escape include-raw001-hello-world.sh + - shell: + !include-raw-escape include-raw001-vars.sh + +- project: + name: test-job-template-1 + num: 1 + jobs: + - 'test-job-include-raw-{num}' diff --git a/tests/localyaml/fixtures/deprecated-include-raw001.json b/tests/localyaml/fixtures/deprecated-include-raw001.json new file mode 100644 index 000000000..e9540524d --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include-raw001.json @@ -0,0 +1,15 @@ +[ + { + "job": { + "name": "test-job-include-raw-1", + "builders": [ + { + "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n" + }, + { + "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n\n" + } + ] + } + } +] diff --git a/tests/localyaml/fixtures/deprecated-include-raw001.yaml b/tests/localyaml/fixtures/deprecated-include-raw001.yaml new file mode 100644 index 000000000..ba2f8ef0c --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include-raw001.yaml @@ -0,0 +1,7 @@ +- job: + name: test-job-include-raw-1 + builders: + - shell: + !include-raw include-raw001-hello-world.sh + - shell: + !include-raw include-raw001-vars.sh diff --git a/tests/localyaml/fixtures/deprecated-include001.json b/tests/localyaml/fixtures/deprecated-include001.json new file mode 100644 index 000000000..3b29d2402 --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include001.json @@ -0,0 +1,43 @@ +[ + { + "job": { + "name": "test-job-1", + "builders": [ + { + "copyartifact": { + "project": "foo", + "filter": "*.tar.gz", + "target": "/home/foo", + "which-build": "last-successful", + "optional": true, + "flatten": true, + "parameter-filters": "PUBLISH=true" + } + }, + { + "copyartifact": { + "project": "bar", + "filter": "*.tar.gz", + "target": "/home/foo", + "which-build": "specific-build", + "optional": true, + "flatten": true, + "parameter-filters": "PUBLISH=true", + "build-number": 123 + } + }, + { + "copyartifact": { + "project": "baz", + "filter": "*.tar.gz", + "target": "/home/foo", + "which-build": "upstream-build", + "optional": true, + "flatten": true, + "parameter-filters": "PUBLISH=true" + } + } + ] + } + } +] diff --git a/tests/localyaml/fixtures/deprecated-include001.yaml b/tests/localyaml/fixtures/deprecated-include001.yaml new file mode 100644 index 000000000..e29a2e9d1 --- /dev/null +++ b/tests/localyaml/fixtures/deprecated-include001.yaml @@ -0,0 +1,4 @@ +- job: + name: test-job-1 + builders: + !include include001.yaml.inc diff --git a/tests/localyaml/fixtures/exception_include001.yaml b/tests/localyaml/fixtures/exception_include001.yaml index 828b541b0..66d9f76e2 100644 --- a/tests/localyaml/fixtures/exception_include001.yaml +++ b/tests/localyaml/fixtures/exception_include001.yaml @@ -12,4 +12,4 @@ - job: name: test-job-1 builders: - !include exception_include001.yaml.inc + !include: exception_include001.yaml.inc diff --git a/tests/localyaml/fixtures/include-raw-escaped001.yaml b/tests/localyaml/fixtures/include-raw-escaped001.yaml index 2853d03ba..df4668282 100644 --- a/tests/localyaml/fixtures/include-raw-escaped001.yaml +++ b/tests/localyaml/fixtures/include-raw-escaped001.yaml @@ -2,9 +2,9 @@ name: test-job-include-raw-{num} builders: - shell: - !include-raw-escape include-raw001-hello-world.sh + !include-raw-escape: include-raw001-hello-world.sh - shell: - !include-raw-escape include-raw001-vars.sh + !include-raw-escape: include-raw001-vars.sh - project: name: test-job-template-1 diff --git a/tests/localyaml/fixtures/include-raw001.yaml b/tests/localyaml/fixtures/include-raw001.yaml index ba2f8ef0c..dfa5fb71f 100644 --- a/tests/localyaml/fixtures/include-raw001.yaml +++ b/tests/localyaml/fixtures/include-raw001.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw include-raw001-hello-world.sh + !include-raw: include-raw001-hello-world.sh - shell: - !include-raw include-raw001-vars.sh + !include-raw: include-raw001-vars.sh diff --git a/tests/localyaml/fixtures/include001.yaml b/tests/localyaml/fixtures/include001.yaml index e29a2e9d1..ed662f8db 100644 --- a/tests/localyaml/fixtures/include001.yaml +++ b/tests/localyaml/fixtures/include001.yaml @@ -1,4 +1,4 @@ - job: name: test-job-1 builders: - !include include001.yaml.inc + !include: include001.yaml.inc diff --git a/tests/yamlparser/fixtures/include002.xml b/tests/yamlparser/fixtures/deprecated-include001.xml similarity index 100% rename from tests/yamlparser/fixtures/include002.xml rename to tests/yamlparser/fixtures/deprecated-include001.xml diff --git a/tests/yamlparser/fixtures/include-raw-escape001.yaml b/tests/yamlparser/fixtures/include-raw-escape001.yaml index abb5b93b5..d41417e4a 100644 --- a/tests/yamlparser/fixtures/include-raw-escape001.yaml +++ b/tests/yamlparser/fixtures/include-raw-escape001.yaml @@ -11,7 +11,7 @@ keep-system-variables: true builders: - shell: - !include-raw-escape include-raw-escape001-echo-vars.sh + !include-raw-escape: include-raw-escape001-echo-vars.sh - project: diff --git a/tests/yamlparser/fixtures/include-raw001.xml b/tests/yamlparser/fixtures/include-raw001.xml index 688a0bb93..494410c5f 100644 --- a/tests/yamlparser/fixtures/include-raw001.xml +++ b/tests/yamlparser/fixtures/include-raw001.xml @@ -19,20 +19,48 @@ - - - #!/bin/bash -# -# Sample script showing how the yaml include-raw tag can be used -# to inline scripts that are maintained outside of the jenkins -# job yaml configuration. - -echo "hello world" - -exit 0 - - - + - + + + 3 + true + false + 150 + 90 + elastic + + + + + #!/bin/bash +echo "Doing somethiung cool" + + + + #!/bin/zsh +echo "Doing somethin cool with zsh" + + + + target1 target2 + Standard Ant + + + + example.prop + EXAMPLE=foo-bar + + + + + + file1,file2*.txt + file2bad.txt + false + false + userContent + false + + diff --git a/tests/yamlparser/fixtures/include-raw001.yaml b/tests/yamlparser/fixtures/include-raw001.yaml index a89f61eaf..bff63ccd4 100644 --- a/tests/yamlparser/fixtures/include-raw001.yaml +++ b/tests/yamlparser/fixtures/include-raw001.yaml @@ -1,10 +1,45 @@ +# vim: sw=4 ts=4 et +- wrapper: + name: timeout-wrapper + wrappers: + - timeout: + fail: true + elastic-percentage: 150 + elastic-default-timeout: 90 + type: elastic + +- wrapper: + name: pre-scm-shell-ant + wrappers: + - pre-scm-buildstep: + - shell: + !include-raw: include-raw002-cool.sh + - shell: + !include-raw: include-raw002-cool.zsh + - ant: + targets: "target1 target2" + ant-name: "Standard Ant" + - inject: + properties-file: example.prop + properties-content: EXAMPLE=foo-bar + +- wrapper: + name: copy-files + wrappers: + - copy-to-slave: + includes: + - file1 + - file2*.txt + excludes: + - file2bad.txt + + - job: - name: test-job-2 + name: test-job-3 + wrappers: + !include: include001.yaml.inc properties: - inject: keep-build-variables: true keep-system-variables: true - builders: - - shell: - !include-raw ../../localyaml/fixtures/include-raw001-hello-world.sh diff --git a/tests/yamlparser/fixtures/include-raw002.xml b/tests/yamlparser/fixtures/include-raw002.xml deleted file mode 100644 index 494410c5f..000000000 --- a/tests/yamlparser/fixtures/include-raw002.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - <!-- Managed by Jenkins Job Builder --> - false - false - false - false - true - - - - false - - true - true - true - false - - - - - - - - 3 - true - false - 150 - 90 - elastic - - - - - #!/bin/bash -echo "Doing somethiung cool" - - - - #!/bin/zsh -echo "Doing somethin cool with zsh" - - - - target1 target2 - Standard Ant - - - - example.prop - EXAMPLE=foo-bar - - - - - - file1,file2*.txt - file2bad.txt - false - false - userContent - false - - - diff --git a/tests/yamlparser/fixtures/include-raw002.yaml b/tests/yamlparser/fixtures/include-raw002.yaml deleted file mode 100644 index cd0f26044..000000000 --- a/tests/yamlparser/fixtures/include-raw002.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# vim: sw=4 ts=4 et -- wrapper: - name: timeout-wrapper - wrappers: - - timeout: - fail: true - elastic-percentage: 150 - elastic-default-timeout: 90 - type: elastic - -- wrapper: - name: pre-scm-shell-ant - wrappers: - - pre-scm-buildstep: - - shell: - !include-raw include-raw002-cool.sh - - shell: - !include-raw include-raw002-cool.zsh - - ant: - targets: "target1 target2" - ant-name: "Standard Ant" - - inject: - properties-file: example.prop - properties-content: EXAMPLE=foo-bar - -- wrapper: - name: copy-files - wrappers: - - copy-to-slave: - includes: - - file1 - - file2*.txt - excludes: - - file2bad.txt - - -- job: - name: test-job-3 - wrappers: - !include include001.yaml.inc - properties: - - inject: - keep-build-variables: true - keep-system-variables: true - diff --git a/tests/yamlparser/fixtures/include-rawunicode001.yaml b/tests/yamlparser/fixtures/include-rawunicode001.yaml index e9c303660..e1379dc68 100644 --- a/tests/yamlparser/fixtures/include-rawunicode001.yaml +++ b/tests/yamlparser/fixtures/include-rawunicode001.yaml @@ -3,7 +3,7 @@ wrappers: - pre-scm-buildstep: - shell: - !include-raw include-rawunicode001-cool.sh + !include-raw: include-rawunicode001-cool.sh - job: name: test-unicode-raw-include-wrapper diff --git a/tests/yamlparser/fixtures/include001.yaml b/tests/yamlparser/fixtures/include001.yaml index 85a54525e..8c7c3cb08 100644 --- a/tests/yamlparser/fixtures/include001.yaml +++ b/tests/yamlparser/fixtures/include001.yaml @@ -38,7 +38,7 @@ - job: name: test-job-1 wrappers: - !include include001.yaml.inc + !include: include001.yaml.inc properties: - inject: keep-build-variables: true diff --git a/tests/yamlparser/fixtures/include002.yaml b/tests/yamlparser/fixtures/include002.yaml deleted file mode 100644 index 13a2f1219..000000000 --- a/tests/yamlparser/fixtures/include002.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# vim: sw=4 ts=4 et -- default: - name: default-timeout-wrapper - timeout: &timeout - fail: true - elastic-percentage: 150 - elastic-default-timeout: 90 - type: elastic - -# include that uses timeout alias -- wrapper: !include include002_1.yaml.inc - -- job: - name: test-job-1 - wrappers: - !include include002.yaml.inc diff --git a/tests/yamlparser/fixtures/include002.yaml.inc b/tests/yamlparser/fixtures/include002.yaml.inc deleted file mode 100644 index e41c6be6c..000000000 --- a/tests/yamlparser/fixtures/include002.yaml.inc +++ /dev/null @@ -1 +0,0 @@ -- timeout-wrapper diff --git a/tests/yamlparser/fixtures/include002_1.yaml.inc b/tests/yamlparser/fixtures/include002_1.yaml.inc deleted file mode 100644 index b127f07b7..000000000 --- a/tests/yamlparser/fixtures/include002_1.yaml.inc +++ /dev/null @@ -1,3 +0,0 @@ -name: timeout-wrapper -wrappers: - - timeout: *timeout diff --git a/tests/yamlparser/fixtures/include_path002.yaml b/tests/yamlparser/fixtures/include_path002.yaml index 70b3f1867..0d273b8bf 100644 --- a/tests/yamlparser/fixtures/include_path002.yaml +++ b/tests/yamlparser/fixtures/include_path002.yaml @@ -5,4 +5,4 @@ - job-template: name: 'include_path' - description: !include test.inc \ No newline at end of file + description: !include: test.inc