Add !include-jinja2 for rendering templates with Jinja2

This template included using !include-jinja2:

"""
{{ my_var }}
"""

is rendered the same as an existing template that looks like this:

"""
{my_var}
"""

This also allows the use of Jinja2's richer syntax:

"""
{% for test_environment in configuration.get("envs", ["py35"]) %}
tox -e {{ test_environment }}
{% endfor %}
"""

Story: 2001135

Change-Id: Ia3ee21822d6e9237f5ea46796bc8810ecac61e2c

jenkins_jobs/formatter.py

@@ -21,6 +21,7 @@ import re
 from string import Formatter
 
 from jenkins_jobs.errors import JenkinsJobsException
+from jenkins_jobs.local_yaml import CustomLoader
 
 logger = logging.getLogger(__name__)
 
@@ -68,6 +69,10 @@ def deep_format(obj, paramdict, allow_empty=False):
                 raise
     else:
         ret = obj
+    if isinstance(ret, CustomLoader):
+        # If we have a CustomLoader here, we've lazily-loaded a template;
+        # attempt to format it.
+        ret = deep_format(ret, paramdict, allow_empty=allow_empty)
     return ret
 
 

jenkins_jobs/local_yaml.py

@@ -127,6 +127,20 @@ Example:
     on any filename passed via ``!include-raw-escape:`` the tag will be
     automatically converted to ``!include-raw:`` and no escaping will be
     performed.
+
+
+The tag ``!include-jinja2:`` will treat the given string or list of strings as
+filenames to be opened as Jinja2 templates, which should be rendered to a
+string and included in the calling YAML construct.  (This is analogous to the
+templating that will happen with ``!include-raw``.)
+
+Examples:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml
+
+    contents of jinja01.yaml.inc:
+
+        .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc
 """
 
 import functools
@@ -135,6 +149,7 @@ import logging
 import os
 import re
 
+import jinja2
 import yaml
 from yaml.constructor import BaseConstructor
 from yaml.representer import BaseRepresenter
@@ -349,8 +364,8 @@ class YamlInclude(BaseYAMLObject):
         elif isinstance(node, yaml.SequenceNode):
             contents = [cls._from_file(loader, scalar_node)
                         for scalar_node in node.value]
-            if any(isinstance(s, LazyLoader) for s in contents):
-                return LazyLoaderCollection(contents)
+            if any(isinstance(s, CustomLoader) for s in contents):
+                return CustomLoaderCollection(contents)
 
             return u'\n'.join(contents)
         else:
@@ -383,6 +398,17 @@ class YamlIncludeRawEscape(YamlIncludeRaw):
             return loader.escape_callback(data)
 
 
+class YamlIncludeJinja2(YamlIncludeRaw):
+    yaml_tag = u'!include-jinja2:'
+
+    @classmethod
+    def _from_file(cls, loader, node):
+        contents = cls._open_file(loader, node)
+        if isinstance(contents, LazyLoader):
+            return contents
+        return Jinja2Loader(contents)
+
+
 class DeprecatedTag(BaseYAMLObject):
 
     @classmethod
@@ -407,8 +433,22 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag):
     _new = YamlIncludeRawEscape
 
 
-class LazyLoaderCollection(object):
-    """Helper class to format a collection of LazyLoader objects"""
+class CustomLoader(object):
+    """Parent class for non-standard loaders."""
+
+
+class Jinja2Loader(CustomLoader):
+    """A loader for Jinja2-templated files."""
+    def __init__(self, contents):
+        self._template = jinja2.Template(contents)
+        self._template.environment.undefined = jinja2.StrictUndefined
+
+    def format(self, **kwargs):
+        return self._template.render(kwargs)
+
+
+class CustomLoaderCollection(object):
+    """Helper class to format a collection of CustomLoader objects"""
     def __init__(self, sequence):
         self._data = sequence
 
@@ -416,7 +456,7 @@ class LazyLoaderCollection(object):
         return u'\n'.join(item.format(*args, **kwargs) for item in self._data)
 
 
-class LazyLoader(object):
+class LazyLoader(CustomLoader):
     """Helper class to provide lazy loading of files included using !include*
     tags where the path to the given file contains unresolved placeholders.
     """

requirements.txt

@@ -7,3 +7,4 @@ pbr>=1.8 # Apache-2.0
 stevedore>=1.17.1 # Apache-2.0
 python-jenkins>=0.4.8
 fasteners
+Jinja2

tests/yamlparser/fixtures/jinja01.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>test variable
+a
+b
+c
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja01.yaml

@@ -0,0 +1,15 @@
+- builder:
+    name: test-builder
+    builders:
+        - shell:
+            !include-jinja2: jinja01.yaml.inc
+
+- job:
+    name: test-job
+    builders:
+        - test-builder:
+            var: "test variable"
+            test_list:
+                - a
+                - b
+                - c

tests/yamlparser/fixtures/jinja01.yaml.inc

@@ -0,0 +1,4 @@
+{{ var }}
+{% for item in test_list -%}
+{{ item }}
+{% endfor %}

tests/yamlparser/fixtures/jinja02.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>1</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja02.yaml

@@ -0,0 +1,11 @@
+- job-template:
+    name: 'test-job-{num}'
+    builders:
+        - shell:
+            !include-jinja2: jinja02.yaml.inc
+
+- project:
+    name: test-job-template-1
+    num: 1
+    jobs:
+      - 'test-job-{num}'

tests/yamlparser/fixtures/jinja02.yaml.inc

@@ -0,0 +1 @@
+{{ num }}

tests/yamlparser/fixtures/lazy-load-jinja001.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>test variable
+a
+b
+c
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/lazy-load-jinja001.yaml

@@ -0,0 +1,16 @@
+- builder:
+    name: test-builder
+    builders:
+        - shell:
+            !include-jinja2: jinja{include-number}.yaml.inc
+
+- job:
+    name: test-job
+    builders:
+        - test-builder:
+            var: "test variable"
+            test_list:
+                - a
+                - b
+                - c
+            include-number: "01"