Add job_builder option to load extra modules for jinja2 filters

Change-Id: Ie6ec481f7ebefc536fc152ef68f73bdc618c54a1
Signed-off-by: Kienan Stewart <kstewart@efficios.com>

doc/source/execution.rst

@@ -72,6 +72,14 @@ job_builder section
   ``--jobs-only`` or ``--views-only`` CLI options.
   (Valid options: jobs, views, all)
 
+**filter_modules**
+  (Optional) A space-separated set of strings which of names of additional
+  modules to load Jinja2 filters from when templating YAML objects. Note
+  that the modules must be in the Python path. To learn about writing
+  custom Jinja2 filters, please see the `Jinja2 Documentation`__
+
+__ https://jinja.palletsprojects.com/en/latest/api/#writing-filters
+
 jenkins section
 ^^^^^^^^^^^^^^^
 

jenkins_jobs/config.py

@@ -40,6 +40,7 @@ exclude=.*
 allow_duplicates=False
 allow_empty_variables=False
 retain_anchors=False
+filter_modules=
 
 # other named sections could be used in addition to the implicit [jenkins]
 # if you have multiple jenkins servers.
@@ -302,6 +303,17 @@ class JJBConfig(object):
             path = config.get("job_builder", "include_path").split(":")
         self.yamlparser["include_path"] = path
 
+        # Extra modules to load for jinja2 filters
+        filter_modules = []
+        if (
+            config
+            and config.has_section("job_builder")
+            and config.has_option("job_builder", "filter_modules")
+            and config.get("job_builder", "filter_modules")
+        ):
+            filter_modules = config.get("job_builder", "filter_modules").split(" ")
+        self.yamlparser["filter_modules"] = filter_modules
+
         # allow duplicates?
         allow_duplicates = False
         if config and config.has_option("job_builder", "allow_duplicates"):

jenkins_jobs/yaml_objects.py

@@ -170,6 +170,7 @@ Examples:
 """
 
 import abc
+import importlib
 import logging
 import traceback
 import sys
@@ -221,6 +222,7 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
         if loader.source_dir:
             # Loaded from a file, find includes beside it too.
             self._search_path.append(loader.source_dir)
+        self._filter_modules = jjb_config.yamlparser["filter_modules"].copy()
         self._loader = loader
         self._pos = pos
         allow_empty = jjb_config.yamlparser["allow_empty_variables"]
@@ -255,10 +257,15 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
 class J2BaseYamlObject(BaseYamlObject):
     def __init__(self, jjb_config, loader, pos):
         super().__init__(jjb_config, loader, pos)
+        self._filters = {}
+        for module_name in self._filter_modules:
+            module = importlib.import_module(module_name)
+            self._filters.update(module.FILTERS)
         self._jinja2_env = jinja2.Environment(
             loader=jinja2.FileSystemLoader(self._search_path),
             undefined=jinja2.StrictUndefined,
         )
+        self._jinja2_env.filters.update(self._filters)
 
     def _render_template(self, pos, template_text, template, params):
         try:

tests/cmd/fixtures/cmd-003.conf

@@ -0,0 +1,2 @@
+[job_builder]
+filter_modules=my_filter my_other_filter

tests/cmd/test_config.py

@@ -175,3 +175,17 @@ def test_update_timeout_set(mocker, fixtures_dir):
 
     jjb_config = jenkins_mock.call_args[0][0]
     assert jjb_config.jenkins["timeout"] == 0.2
+
+
+def test_filter_modules_set(mocker, fixtures_dir):
+    """
+    Check that customs filters modules are set.
+
+    Test that the filter_modules option is a non-empty list.
+    """
+    config_file = fixtures_dir / "cmd-003.conf"
+    args = ["--conf", str(config_file), "test", "foo"]
+    jenkins_jobs = entry.JenkinsJobs(args)
+
+    jjb_config = jenkins_jobs.jjb_config
+    assert jjb_config.yamlparser["filter_modules"] == ["my_filter", "my_other_filter"]

tests/yamlparser/job_fixtures/filter_modules.conf

@@ -0,0 +1,2 @@
+[job_builder]
+filter_modules=myfilter

tests/yamlparser/job_fixtures/filter_modules.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>my_filter says &quot;hello&quot;</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/job_fixtures/filter_modules.yaml

@@ -0,0 +1,11 @@
+- project:
+    name: filter_modules
+    jobs:
+        - 'filter_modules'
+
+- job-template:
+    name: 'filter_modules'
+    builders:
+      - shell:
+          !j2: |
+            {{"hello"|my_filter}}

tests/yamlparser/job_fixtures/myfilter/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+"""
+Example custom jinja2 filter
+"""
+
+import jinja2
+
+
+@jinja2.pass_environment
+def do_my_filter(env, data, skip_list_wrap=False):
+    return 'my_filter says "{}"'.format(data)
+
+
+FILTERS = {
+    "my_filter": do_my_filter,
+}

tests/yamlparser/test_jobs.py

@@ -18,6 +18,7 @@
 import os
 from operator import attrgetter
 from pathlib import Path
+import sys
 
 import pytest
 
@@ -36,6 +37,9 @@ def scenario(request):
 
 
 def test_yaml_snippet(scenario, check_job):
+    old_path = sys.path
+    if str(fixtures_dir) not in sys.path:
+        sys.path.append(str(fixtures_dir))
     # Some tests using config with 'include_path' expect JJB root to be current directory.
     os.chdir(Path(__file__).parent / "../..")
     if scenario.name.startswith("deprecated-"):
@@ -44,3 +48,4 @@ def test_yaml_snippet(scenario, check_job):
         assert "is deprecated" in str(record[0].message)
     else:
         check_job()
+    sys.path = old_path