diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 4bc400ed1..2fd51defe 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -261,6 +261,18 @@ For example: .. literalinclude:: /../../tests/yamlparser/fixtures/second_order_parameter_interpolation002.yaml +By default JJB will fail if it tries to interpolate a variable that was not +defined, but you can change that behaviour and allow empty variables with the +allow_empty_variables configuration option. + +For example, having a configuration file with tha toption enabled: + +.. literalinclude:: /../../tests/yamlparser/fixtures/allow_empty_variables.conf + +Will prevent JJb from failing if there are any non-initialized variables used +and replace them with the empty string instead. + + Yaml Anchors & Aliases ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 5c63ee722..7dd3d9a10 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -88,6 +88,14 @@ job_builder section correct one to use. When this option is set to True, only a warning is emitted. +**allow_empty_variables** + (Optional) When expanding strings, by default `jenkins-jobs` will raise an + exception if there's a key in the string, that has not been declared on the + yamls. Setting this options to True, will replace it with the empty string, + allowing you to use those strings without having to define all the keys it + might be using. + + jenkins section ^^^^^^^^^^^^^^^ diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index e056cc395..34b500e41 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -32,6 +32,7 @@ import logging import copy import itertools import fnmatch +from string import Formatter from jenkins_jobs.errors import JenkinsJobsException import jenkins_jobs.local_yaml as local_yaml @@ -39,6 +40,28 @@ logger = logging.getLogger(__name__) MAGIC_MANAGE_STRING = "" +class CustomFormatter(Formatter): + """ + Custom formatter to allow non-existing key references when formatting a + string + """ + def __init__(self, allow_empty=False): + super(CustomFormatter, self).__init__() + self.allow_empty = allow_empty + + def get_value(self, key, args, kwargs): + try: + return Formatter.get_value(self, key, args, kwargs) + except KeyError: + if self.allow_empty: + logger.debug( + 'Found uninitialized key %s, replaced with empty string', + key + ) + return '' + raise + + # Python 2.6's minidom toprettyxml produces broken output by adding extraneous # whitespace around data. This patches the broken implementation with one taken # from Python > 2.7.3 @@ -76,7 +99,7 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml': minidom.Element.writexml = writexml -def deep_format(obj, paramdict): +def deep_format(obj, paramdict, allow_empty=False): """Apply the paramdict via str.format() to all string objects found within the supplied obj. Lists and dicts are traversed recursively.""" # YAML serialisation was originally used to achieve this, but that places @@ -89,22 +112,22 @@ def deep_format(obj, paramdict): if result is not None: ret = paramdict[result.group("key")] else: - ret = obj.format(**paramdict) + ret = CustomFormatter(allow_empty).format(obj, **paramdict) except KeyError as exc: missing_key = exc.message desc = "%s parameter missing to format %s\nGiven:\n%s" % ( - missing_key, obj, pformat(paramdict)) + missing_key, obj, pformat(paramdict)) raise JenkinsJobsException(desc) elif isinstance(obj, list): ret = [] for item in obj: - ret.append(deep_format(item, paramdict)) + ret.append(deep_format(item, paramdict, allow_empty)) elif isinstance(obj, dict): ret = {} for item in obj: try: - ret[item.format(**paramdict)] = \ - deep_format(obj[item], paramdict) + ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \ + deep_format(obj[item], paramdict, allow_empty) except KeyError as exc: missing_key = exc.message desc = "%s parameter missing to format %s\nGiven:\n%s" % ( @@ -364,7 +387,13 @@ class YamlParser(object): params.update(expanded_values) params = deep_format(params, params) - expanded = deep_format(template, params) + allow_empty_variables = self.config \ + and self.config.has_section('job_builder') \ + and self.config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.config.getboolean( + 'job_builder', 'allow_empty_variables') + expanded = deep_format(template, params, allow_empty_variables) job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): @@ -526,7 +555,14 @@ class ModuleRegistry(object): # Template data contains values that should be interpolated # into the component definition s = yaml.dump(component_data, default_flow_style=False) - s = s.format(**template_data) + allow_empty_variables = self.global_config \ + and self.global_config.has_section('job_builder') \ + and self.global_config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.global_config.getboolean( + 'job_builder', 'allow_empty_variables') + s = CustomFormatter( + allow_empty_variables).format(s, **template_data) component_data = yaml.load(s) else: # The component is a simple string name, eg "run-tests" diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index 780d46244..8c1105994 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -36,6 +36,7 @@ ignore_cache=False recursive=False exclude=.* allow_duplicates=False +allow_empty_variables=False [jenkins] url=http://localhost:8080/ @@ -144,6 +145,11 @@ def create_parser(): parser.add_argument('--version', dest='version', action='version', version=version(), help='show version') + parser.add_argument( + '--allow-empty-variables', action='store_true', + dest='allow_empty_variables', default=None, + help='Don\'t fail if any of the variables inside any string are not ' + 'defined, replace with empty string instead') return parser @@ -232,6 +238,10 @@ def execute(options, config): if not isinstance(plugins_info, list): raise JenkinsJobsException("{0} must contain a Yaml list!" .format(options.plugins_info_path)) + if options.allow_empty_variables is not None: + config.set('job_builder', + 'allow_empty_variables', + str(options.allow_empty_variables)) builder = Builder(config.get('jenkins', 'url'), user, diff --git a/tests/yamlparser/fixtures/allow_empty_variables.conf b/tests/yamlparser/fixtures/allow_empty_variables.conf new file mode 100644 index 000000000..2026a6c85 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_empty_variables = True diff --git a/tests/yamlparser/fixtures/allow_empty_variables.xml b/tests/yamlparser/fixtures/allow_empty_variables.xml new file mode 100644 index 000000000..89ace80da --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.xml @@ -0,0 +1,20 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + echo "This should be empty: " + + + + + + diff --git a/tests/yamlparser/fixtures/allow_empty_variables.yaml b/tests/yamlparser/fixtures/allow_empty_variables.yaml new file mode 100644 index 000000000..d8ac3ca69 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables.yaml @@ -0,0 +1,10 @@ +- project: + name: allow_empty_variables + jobs: + - 'allow_empty_variables' + +- job-template: + name: 'allow_empty_variables' + builders: + - shell: | + echo "This should be empty: {my_empty_var}" diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.conf b/tests/yamlparser/fixtures/allow_empty_variables_include.conf new file mode 100644 index 000000000..391bd263c --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.conf @@ -0,0 +1,2 @@ +[job_builder] +allow_empty_variables = true diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.sh b/tests/yamlparser/fixtures/allow_empty_variables_include.sh new file mode 100644 index 000000000..e498a62e2 --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.sh @@ -0,0 +1 @@ +echo "Here ->{myvar}<- you should see nothing" diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.xml b/tests/yamlparser/fixtures/allow_empty_variables_include.xml new file mode 100644 index 000000000..b8d1fd2cb --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.xml @@ -0,0 +1,19 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + echo "Here -><- you should see nothing" + + + + + diff --git a/tests/yamlparser/fixtures/allow_empty_variables_include.yaml b/tests/yamlparser/fixtures/allow_empty_variables_include.yaml new file mode 100644 index 000000000..f8b2662ae --- /dev/null +++ b/tests/yamlparser/fixtures/allow_empty_variables_include.yaml @@ -0,0 +1,11 @@ +- project: + name: allow_empty_variables_include + jobs: + - 'allow_empty_variables_include' + +- job-template: + name: allow_empty_variables_include + builders: + - shell: + !include ./allow_empty_variables_include.sh +