From 814ba7575ff4987492bb40f16c942989f80e02c5 Mon Sep 17 00:00:00 2001 From: David Caro Date: Fri, 20 Jun 2014 17:27:05 +0200 Subject: [PATCH] Added possibility to use non-existent keys Added a new configuration option under the section "job_builder" named "allow_empty_variables" that if set to true, will replace non existing variables in strings with the empty string instead of raising an error. It's very useful if you have a shell script, that has optional values, for example: EXTRA_PACKAGES=({extra-packages}) for package in "${EXTRA_PACKAGES[@]}"; do install "$package" done Or modifying the script behavior with flags, with empty as default value: WITH_AUTOGEN={with-autogen} if [[ $WITH_AUTOGEN ]]; then ./autogen.sh fi Then if you have two different jobs that use that script in their builder, you just have to set the extra parameter or ignore it to change the behavior: - builder: name: mybuilder builders: - shell: !include shell-scripts/myscript.sh - job-template: name: 'mytpl-{name}' ... builders: - mybuilder - project: name: myproj1 with-autogen: true jobs: - 'mytpl-{name}' - project: name: myproj2 extra-packages: | extrapkg1 extrapkg2 Change-Id: Iad9f0e522725e6fd6681cd62d3e36f69baf09585 Signed-off-by: David Caro --- doc/source/definition.rst | 12 +++++ doc/source/installation.rst | 8 +++ jenkins_jobs/builder.py | 52 ++++++++++++++++--- jenkins_jobs/cmd.py | 10 ++++ .../fixtures/allow_empty_variables.conf | 2 + .../fixtures/allow_empty_variables.xml | 20 +++++++ .../fixtures/allow_empty_variables.yaml | 10 ++++ .../allow_empty_variables_include.conf | 2 + .../fixtures/allow_empty_variables_include.sh | 1 + .../allow_empty_variables_include.xml | 19 +++++++ .../allow_empty_variables_include.yaml | 11 ++++ 11 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 tests/yamlparser/fixtures/allow_empty_variables.conf create mode 100644 tests/yamlparser/fixtures/allow_empty_variables.xml create mode 100644 tests/yamlparser/fixtures/allow_empty_variables.yaml create mode 100644 tests/yamlparser/fixtures/allow_empty_variables_include.conf create mode 100644 tests/yamlparser/fixtures/allow_empty_variables_include.sh create mode 100644 tests/yamlparser/fixtures/allow_empty_variables_include.xml create mode 100644 tests/yamlparser/fixtures/allow_empty_variables_include.yaml diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 4bc400ed1..2fd51defe 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -261,6 +261,18 @@ For example: .. literalinclude:: /../../tests/yamlparser/fixtures/second_order_parameter_interpolation002.yaml +By default JJB will fail if it tries to interpolate a variable that was not +defined, but you can change that behaviour and allow empty variables with the +allow_empty_variables configuration option. + +For example, having a configuration file with tha toption enabled: + +.. literalinclude:: /../../tests/yamlparser/fixtures/allow_empty_variables.conf + +Will prevent JJb from failing if there are any non-initialized variables used +and replace them with the empty string instead. + + Yaml Anchors & Aliases ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 5c63ee722..7dd3d9a10 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -88,6 +88,14 @@ job_builder section correct one to use. When this option is set to True, only a warning is emitted. +**allow_empty_variables** + (Optional) When expanding strings, by default `jenkins-jobs` will raise an + exception if there's a key in the string, that has not been declared on the + yamls. Setting this options to True, will replace it with the empty string, + allowing you to use those strings without having to define all the keys it + might be using. + + jenkins section ^^^^^^^^^^^^^^^ diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index e056cc395..34b500e41 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -32,6 +32,7 @@ import logging import copy import itertools import fnmatch +from string import Formatter from jenkins_jobs.errors import JenkinsJobsException import jenkins_jobs.local_yaml as local_yaml @@ -39,6 +40,28 @@ logger = logging.getLogger(__name__) MAGIC_MANAGE_STRING = "" +class CustomFormatter(Formatter): + """ + Custom formatter to allow non-existing key references when formatting a + string + """ + def __init__(self, allow_empty=False): + super(CustomFormatter, self).__init__() + self.allow_empty = allow_empty + + def get_value(self, key, args, kwargs): + try: + return Formatter.get_value(self, key, args, kwargs) + except KeyError: + if self.allow_empty: + logger.debug( + 'Found uninitialized key %s, replaced with empty string', + key + ) + return '' + raise + + # Python 2.6's minidom toprettyxml produces broken output by adding extraneous # whitespace around data. This patches the broken implementation with one taken # from Python > 2.7.3 @@ -76,7 +99,7 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml': minidom.Element.writexml = writexml -def deep_format(obj, paramdict): +def deep_format(obj, paramdict, allow_empty=False): """Apply the paramdict via str.format() to all string objects found within the supplied obj. Lists and dicts are traversed recursively.""" # YAML serialisation was originally used to achieve this, but that places @@ -89,22 +112,22 @@ def deep_format(obj, paramdict): if result is not None: ret = paramdict[result.group("key")] else: - ret = obj.format(**paramdict) + ret = CustomFormatter(allow_empty).format(obj, **paramdict) except KeyError as exc: missing_key = exc.message desc = "%s parameter missing to format %s\nGiven:\n%s" % ( - missing_key, obj, pformat(paramdict)) + missing_key, obj, pformat(paramdict)) raise JenkinsJobsException(desc) elif isinstance(obj, list): ret = [] for item in obj: - ret.append(deep_format(item, paramdict)) + ret.append(deep_format(item, paramdict, allow_empty)) elif isinstance(obj, dict): ret = {} for item in obj: try: - ret[item.format(**paramdict)] = \ - deep_format(obj[item], paramdict) + ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \ + deep_format(obj[item], paramdict, allow_empty) except KeyError as exc: missing_key = exc.message desc = "%s parameter missing to format %s\nGiven:\n%s" % ( @@ -364,7 +387,13 @@ class YamlParser(object): params.update(expanded_values) params = deep_format(params, params) - expanded = deep_format(template, params) + allow_empty_variables = self.config \ + and self.config.has_section('job_builder') \ + and self.config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.config.getboolean( + 'job_builder', 'allow_empty_variables') + expanded = deep_format(template, params, allow_empty_variables) job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): @@ -526,7 +555,14 @@ class ModuleRegistry(object): # Template data contains values that should be interpolated # into the component definition s = yaml.dump(component_data, default_flow_style=False) - s = s.format(**template_data) + allow_empty_variables = self.global_config \ + and self.global_config.has_section('job_builder') \ + and self.global_config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.global_config.getboolean( + 'job_builder', 'allow_empty_variables') + s = CustomFormatter( + allow_empty_variables).format(s, **template_data) component_data = yaml.load(s) else: # The component is a simple string name, eg "run-tests" diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index 780d46244..8c1105994 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -36,6 +36,7 @@ ignore_cache=False recursive=False exclude=.* allow_duplicates=False +allow_empty_variables=False [jenkins] url=http://localhost:8080/ @@ -144,6 +145,11 @@ def create_parser(): parser.add_argument('--version', dest='version', action='version', version=version(), help='show version') + parser.add_argument( + '--allow-empty-variables', action='store_true', + dest='allow_empty_variables', default=None, + help='Don\'t fail if any of the variables inside any string are not ' + 'defined, replace with empty string instead') return parser @@ -232,6 +238,10 @@ def execute(options, config): if not isinstance(plugins_info, list): raise JenkinsJobsException("{0} must contain a Yaml list!" .format(options.plugins_info_path)) + if options.allow_empty_variables is not None: + config.set('job_builder', + 'allow_empty_variables', + str(options.allow_empty_variables)) builder = Builder(config.get('jenkins', 'url'), user, diff --git a/tests/yamlparser/fixtures/allow_empty_variables.conf b/tests/yamlparser/fixtures/allow_empty_variables.conf new file mode 100644 index 000000000..2026a6c85 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_empty_variables = True diff --git a/tests/yamlparser/fixtures/allow_empty_variables.xml b/tests/yamlparser/fixtures/allow_empty_variables.xml new file mode 100644 index 000000000..89ace80da --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.xml @@ -0,0 +1,20 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + echo "This should be empty: " + + + + + + diff --git a/tests/yamlparser/fixtures/allow_empty_variables.yaml b/tests/yamlparser/fixtures/allow_empty_variables.yaml new file mode 100644 index 000000000..d8ac3ca69 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.yaml @@ -0,0 +1,10 @@ +- project: + name: allow_empty_variables + jobs: + - 'allow_empty_variables' + +- job-template: + name: 'allow_empty_variables' + builders: + - shell: | + echo "This should be empty: {my_empty_var}" diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.conf b/tests/yamlparser/fixtures/allow_empty_variables_include.conf new file mode 100644 index 000000000..391bd263c --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_empty_variables = true diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.sh b/tests/yamlparser/fixtures/allow_empty_variables_include.sh new file mode 100644 index 000000000..e498a62e2 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.sh @@ -0,0 +1 @@ +echo "Here ->{myvar}<- you should see nothing" diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.xml b/tests/yamlparser/fixtures/allow_empty_variables_include.xml new file mode 100644 index 000000000..b8d1fd2cb --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.xml @@ -0,0 +1,19 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + echo "Here -><- you should see nothing" + + + + + diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.yaml b/tests/yamlparser/fixtures/allow_empty_variables_include.yaml new file mode 100644 index 000000000..f8b2662ae --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.yaml @@ -0,0 +1,11 @@ +- project: + name: allow_empty_variables_include + jobs: + - 'allow_empty_variables_include' + +- job-template: + name: allow_empty_variables_include + builders: + - shell: + !include ./allow_empty_variables_include.sh +