Remove YamlParser from jenkins_jobs.builder
The goal of this patch is simply to move some classes out of jenkins_jobs.builder into more appropriately-named modules. This started with simply moving YamlParser into jenkins_jobs.parser but led to other moves in order to avoid cyclic imports since YamlParser uses other classes previously defined in jenkins_jobs.builder. That said, this patch doesn't intend to address all of the clutter in jenkins_jobs.builder, mostly just what is necessary to get started working on YamlParser independent of other classes in that module. Change-Id: Ie88bf683e495033eb0b670fe29c256a70282735f
This commit is contained in:
Wayne
parent
c29066210c
commit
4a8b93b8c2
@ -83,5 +83,5 @@ All modules and their associated components are registered in the
|
||||
module registry. It can be accessed either from modules via the registry
|
||||
field, or via the parser parameter of components.
|
||||
|
||||
.. autoclass:: jenkins_jobs.builder.ModuleRegistry
|
||||
.. autoclass:: jenkins_jobs.registry.ModuleRegistry
|
||||
:members:
|
||||
|
||||
@ -26,40 +26,13 @@ import xml
|
||||
from xml.dom import minidom
|
||||
import jenkins
|
||||
import re
|
||||
import pkg_resources
|
||||
from pprint import pformat
|
||||
import logging
|
||||
import copy
|
||||
import itertools
|
||||
import fnmatch
|
||||
from string import Formatter
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
import jenkins_jobs.local_yaml as local_yaml
|
||||
|
||||
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
|
||||
from jenkins_jobs.parser import YamlParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
|
||||
|
||||
|
||||
class CustomFormatter(Formatter):
|
||||
"""
|
||||
Custom formatter to allow non-existing key references when formatting a
|
||||
string
|
||||
"""
|
||||
def __init__(self, allow_empty=False):
|
||||
super(CustomFormatter, self).__init__()
|
||||
self.allow_empty = allow_empty
|
||||
|
||||
def get_value(self, key, args, kwargs):
|
||||
try:
|
||||
return Formatter.get_value(self, key, args, kwargs)
|
||||
except KeyError:
|
||||
if self.allow_empty:
|
||||
logger.debug(
|
||||
'Found uninitialized key %s, replaced with empty string',
|
||||
key
|
||||
)
|
||||
return ''
|
||||
raise
|
||||
|
||||
|
||||
# Python 2.6's minidom toprettyxml produces broken output by adding extraneous
|
||||
@ -99,526 +72,6 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml':
|
||||
minidom.Element.writexml = writexml
|
||||
|
||||
|
||||
def deep_format(obj, paramdict, allow_empty=False):
|
||||
"""Apply the paramdict via str.format() to all string objects found within
|
||||
the supplied obj. Lists and dicts are traversed recursively."""
|
||||
# YAML serialisation was originally used to achieve this, but that places
|
||||
# limitations on the values in paramdict - the post-format result must
|
||||
# still be valid YAML (so substituting-in a string containing quotes, for
|
||||
# example, is problematic).
|
||||
if hasattr(obj, 'format'):
|
||||
try:
|
||||
result = re.match('^{obj:(?P<key>\w+)}$', obj)
|
||||
if result is not None:
|
||||
ret = paramdict[result.group("key")]
|
||||
else:
|
||||
ret = CustomFormatter(allow_empty).format(obj, **paramdict)
|
||||
except KeyError as exc:
|
||||
missing_key = exc.message
|
||||
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
|
||||
missing_key, obj, pformat(paramdict))
|
||||
raise JenkinsJobsException(desc)
|
||||
elif isinstance(obj, list):
|
||||
ret = []
|
||||
for item in obj:
|
||||
ret.append(deep_format(item, paramdict, allow_empty))
|
||||
elif isinstance(obj, dict):
|
||||
ret = {}
|
||||
for item in obj:
|
||||
try:
|
||||
ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
|
||||
deep_format(obj[item], paramdict, allow_empty)
|
||||
except KeyError as exc:
|
||||
missing_key = exc.message
|
||||
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
|
||||
missing_key, obj, pformat(paramdict))
|
||||
raise JenkinsJobsException(desc)
|
||||
else:
|
||||
ret = obj
|
||||
return ret
|
||||
|
||||
|
||||
def matches(what, glob_patterns):
|
||||
"""
|
||||
Checks if the given string, ``what``, matches any of the glob patterns in
|
||||
the iterable, ``glob_patterns``
|
||||
|
||||
:arg str what: String that we want to test if it matches a pattern
|
||||
:arg iterable glob_patterns: glob patterns to match (list, tuple, set,
|
||||
etc.)
|
||||
"""
|
||||
return any(fnmatch.fnmatch(what, glob_pattern)
|
||||
for glob_pattern in glob_patterns)
|
||||
|
||||
|
||||
class YamlParser(object):
|
||||
def __init__(self, config=None, plugins_info=None):
|
||||
self.data = {}
|
||||
self.jobs = []
|
||||
self.xml_jobs = []
|
||||
self.config = config
|
||||
self.registry = ModuleRegistry(self.config, plugins_info)
|
||||
self.path = ["."]
|
||||
if self.config:
|
||||
if config.has_section('job_builder') and \
|
||||
config.has_option('job_builder', 'include_path'):
|
||||
self.path = config.get('job_builder',
|
||||
'include_path').split(':')
|
||||
self.keep_desc = self.get_keep_desc()
|
||||
|
||||
def get_keep_desc(self):
|
||||
keep_desc = False
|
||||
if self.config and self.config.has_section('job_builder') and \
|
||||
self.config.has_option('job_builder', 'keep_descriptions'):
|
||||
keep_desc = self.config.getboolean('job_builder',
|
||||
'keep_descriptions')
|
||||
return keep_desc
|
||||
|
||||
def parse_fp(self, fp):
|
||||
data = local_yaml.load(fp, search_path=self.path)
|
||||
if data:
|
||||
if not isinstance(data, list):
|
||||
raise JenkinsJobsException(
|
||||
"The topmost collection in file '{fname}' must be a list,"
|
||||
" not a {cls}".format(fname=getattr(fp, 'name', fp),
|
||||
cls=type(data)))
|
||||
for item in data:
|
||||
cls, dfn = next(iter(item.items()))
|
||||
group = self.data.get(cls, {})
|
||||
if len(item.items()) > 1:
|
||||
n = None
|
||||
for k, v in item.items():
|
||||
if k == "name":
|
||||
n = v
|
||||
break
|
||||
# Syntax error
|
||||
raise JenkinsJobsException("Syntax error, for item "
|
||||
"named '{0}'. Missing indent?"
|
||||
.format(n))
|
||||
name = dfn['name']
|
||||
if name in group:
|
||||
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
|
||||
"already defined".format(fp.name, name))
|
||||
group[name] = dfn
|
||||
self.data[cls] = group
|
||||
|
||||
def parse(self, fn):
|
||||
with open(fn) as fp:
|
||||
self.parse_fp(fp)
|
||||
|
||||
def _handle_dups(self, message):
|
||||
|
||||
if not (self.config and self.config.has_section('job_builder') and
|
||||
self.config.getboolean('job_builder', 'allow_duplicates')):
|
||||
logger.error(message)
|
||||
raise JenkinsJobsException(message)
|
||||
else:
|
||||
logger.warn(message)
|
||||
|
||||
def getJob(self, name):
|
||||
job = self.data.get('job', {}).get(name, None)
|
||||
if not job:
|
||||
return job
|
||||
return self.applyDefaults(job)
|
||||
|
||||
def getJobGroup(self, name):
|
||||
return self.data.get('job-group', {}).get(name, None)
|
||||
|
||||
def getJobTemplate(self, name):
|
||||
job = self.data.get('job-template', {}).get(name, None)
|
||||
if not job:
|
||||
return job
|
||||
return self.applyDefaults(job)
|
||||
|
||||
def applyDefaults(self, data, override_dict=None):
|
||||
if override_dict is None:
|
||||
override_dict = {}
|
||||
|
||||
whichdefaults = data.get('defaults', 'global')
|
||||
defaults = copy.deepcopy(self.data.get('defaults',
|
||||
{}).get(whichdefaults, {}))
|
||||
if defaults == {} and whichdefaults != 'global':
|
||||
raise JenkinsJobsException("Unknown defaults set: '{0}'"
|
||||
.format(whichdefaults))
|
||||
|
||||
for key in override_dict.keys():
|
||||
if key in defaults.keys():
|
||||
defaults[key] = override_dict[key]
|
||||
|
||||
newdata = {}
|
||||
newdata.update(defaults)
|
||||
newdata.update(data)
|
||||
return newdata
|
||||
|
||||
def formatDescription(self, job):
|
||||
if self.keep_desc:
|
||||
description = job.get("description", None)
|
||||
else:
|
||||
description = job.get("description", '')
|
||||
if description is not None:
|
||||
job["description"] = description + \
|
||||
self.get_managed_string().lstrip()
|
||||
|
||||
def expandYaml(self, jobs_glob=None):
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for module in self.registry.modules:
|
||||
if hasattr(module, 'handle_data'):
|
||||
if module.handle_data(self):
|
||||
changed = True
|
||||
|
||||
for job in self.data.get('job', {}).values():
|
||||
if jobs_glob and not matches(job['name'], jobs_glob):
|
||||
logger.debug("Ignoring job {0}".format(job['name']))
|
||||
continue
|
||||
logger.debug("Expanding job '{0}'".format(job['name']))
|
||||
job = self.applyDefaults(job)
|
||||
self.formatDescription(job)
|
||||
self.jobs.append(job)
|
||||
for project in self.data.get('project', {}).values():
|
||||
logger.debug("Expanding project '{0}'".format(project['name']))
|
||||
# use a set to check for duplicate job references in projects
|
||||
seen = set()
|
||||
for jobspec in project.get('jobs', []):
|
||||
if isinstance(jobspec, dict):
|
||||
# Singleton dict containing dict of job-specific params
|
||||
jobname, jobparams = next(iter(jobspec.items()))
|
||||
if not isinstance(jobparams, dict):
|
||||
jobparams = {}
|
||||
else:
|
||||
jobname = jobspec
|
||||
jobparams = {}
|
||||
job = self.getJob(jobname)
|
||||
if job:
|
||||
# Just naming an existing defined job
|
||||
if jobname in seen:
|
||||
self._handle_dups("Duplicate job '{0}' specified "
|
||||
"for project '{1}'".format(
|
||||
jobname, project['name']))
|
||||
seen.add(jobname)
|
||||
continue
|
||||
# see if it's a job group
|
||||
group = self.getJobGroup(jobname)
|
||||
if group:
|
||||
for group_jobspec in group['jobs']:
|
||||
if isinstance(group_jobspec, dict):
|
||||
group_jobname, group_jobparams = \
|
||||
next(iter(group_jobspec.items()))
|
||||
if not isinstance(group_jobparams, dict):
|
||||
group_jobparams = {}
|
||||
else:
|
||||
group_jobname = group_jobspec
|
||||
group_jobparams = {}
|
||||
job = self.getJob(group_jobname)
|
||||
if job:
|
||||
if group_jobname in seen:
|
||||
self._handle_dups(
|
||||
"Duplicate job '{0}' specified for "
|
||||
"project '{1}'".format(group_jobname,
|
||||
project['name']))
|
||||
seen.add(group_jobname)
|
||||
continue
|
||||
template = self.getJobTemplate(group_jobname)
|
||||
# Allow a group to override parameters set by a project
|
||||
d = {}
|
||||
d.update(project)
|
||||
d.update(jobparams)
|
||||
d.update(group)
|
||||
d.update(group_jobparams)
|
||||
# Except name, since the group's name is not useful
|
||||
d['name'] = project['name']
|
||||
if template:
|
||||
self.expandYamlForTemplateJob(d, template,
|
||||
jobs_glob)
|
||||
continue
|
||||
# see if it's a template
|
||||
template = self.getJobTemplate(jobname)
|
||||
if template:
|
||||
d = {}
|
||||
d.update(project)
|
||||
d.update(jobparams)
|
||||
self.expandYamlForTemplateJob(d, template, jobs_glob)
|
||||
else:
|
||||
raise JenkinsJobsException("Failed to find suitable "
|
||||
"template named '{0}'"
|
||||
.format(jobname))
|
||||
# check for duplicate generated jobs
|
||||
seen = set()
|
||||
# walk the list in reverse so that last definition wins
|
||||
for job in self.jobs[::-1]:
|
||||
if job['name'] in seen:
|
||||
self._handle_dups("Duplicate definitions for job '{0}' "
|
||||
"specified".format(job['name']))
|
||||
self.jobs.remove(job)
|
||||
seen.add(job['name'])
|
||||
|
||||
def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
|
||||
dimensions = []
|
||||
template_name = template['name']
|
||||
# reject keys that are not useful during yaml expansion
|
||||
for k in ['jobs']:
|
||||
project.pop(k)
|
||||
for (k, v) in project.items():
|
||||
tmpk = '{{{0}}}'.format(k)
|
||||
if tmpk not in template_name:
|
||||
logger.debug("Variable %s not in name %s, rejecting from job"
|
||||
" matrix expansion.", tmpk, template_name)
|
||||
continue
|
||||
if type(v) == list:
|
||||
dimensions.append(zip([k] * len(v), v))
|
||||
# XXX somewhat hackish to ensure we actually have a single
|
||||
# pass through the loop
|
||||
if len(dimensions) == 0:
|
||||
dimensions = [(("", ""),)]
|
||||
|
||||
for values in itertools.product(*dimensions):
|
||||
params = copy.deepcopy(project)
|
||||
params = self.applyDefaults(params, template)
|
||||
|
||||
expanded_values = {}
|
||||
for (k, v) in values:
|
||||
if isinstance(v, dict):
|
||||
inner_key = next(iter(v))
|
||||
expanded_values[k] = inner_key
|
||||
expanded_values.update(v[inner_key])
|
||||
else:
|
||||
expanded_values[k] = v
|
||||
|
||||
params.update(expanded_values)
|
||||
params = deep_format(params, params)
|
||||
allow_empty_variables = self.config \
|
||||
and self.config.has_section('job_builder') \
|
||||
and self.config.has_option(
|
||||
'job_builder', 'allow_empty_variables') \
|
||||
and self.config.getboolean(
|
||||
'job_builder', 'allow_empty_variables')
|
||||
expanded = deep_format(template, params, allow_empty_variables)
|
||||
|
||||
job_name = expanded.get('name')
|
||||
if jobs_glob and not matches(job_name, jobs_glob):
|
||||
continue
|
||||
|
||||
self.formatDescription(expanded)
|
||||
self.jobs.append(expanded)
|
||||
|
||||
def get_managed_string(self):
|
||||
# The \n\n is not hard coded, because they get stripped if the
|
||||
# project does not otherwise have a description.
|
||||
return "\n\n" + MAGIC_MANAGE_STRING
|
||||
|
||||
def generateXML(self):
|
||||
for job in self.jobs:
|
||||
self.xml_jobs.append(self.getXMLForJob(job))
|
||||
|
||||
def getXMLForJob(self, data):
|
||||
kind = data.get('project-type', 'freestyle')
|
||||
|
||||
for ep in pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.projects', name=kind):
|
||||
Mod = ep.load()
|
||||
mod = Mod(self.registry)
|
||||
xml = mod.root_xml(data)
|
||||
self.gen_xml(xml, data)
|
||||
job = XmlJob(xml, data['name'])
|
||||
return job
|
||||
|
||||
def gen_xml(self, xml, data):
|
||||
for module in self.registry.modules:
|
||||
if hasattr(module, 'gen_xml'):
|
||||
module.gen_xml(self, xml, data)
|
||||
|
||||
|
||||
class ModuleRegistry(object):
|
||||
entry_points_cache = {}
|
||||
|
||||
def __init__(self, config, plugins_list=None):
|
||||
self.modules = []
|
||||
self.modules_by_component_type = {}
|
||||
self.handlers = {}
|
||||
self.global_config = config
|
||||
|
||||
if plugins_list is None:
|
||||
self.plugins_dict = {}
|
||||
else:
|
||||
self.plugins_dict = self._get_plugins_info_dict(plugins_list)
|
||||
|
||||
for entrypoint in pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.modules'):
|
||||
Mod = entrypoint.load()
|
||||
mod = Mod(self)
|
||||
self.modules.append(mod)
|
||||
self.modules.sort(key=operator.attrgetter('sequence'))
|
||||
if mod.component_type is not None:
|
||||
self.modules_by_component_type[mod.component_type] = mod
|
||||
|
||||
@staticmethod
|
||||
def _get_plugins_info_dict(plugins_list):
|
||||
def mutate_plugin_info(plugin_info):
|
||||
"""
|
||||
We perform mutations on a single member of plugin_info here, then
|
||||
return a dictionary with the longName and shortName of the plugin
|
||||
mapped to its plugin info dictionary.
|
||||
"""
|
||||
version = plugin_info.get('version', '0')
|
||||
plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)',
|
||||
r'\g<1>.preview', version)
|
||||
|
||||
aliases = []
|
||||
for key in ['longName', 'shortName']:
|
||||
value = plugin_info.get(key, None)
|
||||
if value is not None:
|
||||
aliases.append(value)
|
||||
|
||||
plugin_info_dict = {}
|
||||
for name in aliases:
|
||||
plugin_info_dict[name] = plugin_info
|
||||
|
||||
return plugin_info_dict
|
||||
|
||||
list_of_dicts = [mutate_plugin_info(v) for v in plugins_list]
|
||||
|
||||
plugins_info_dict = {}
|
||||
for d in list_of_dicts:
|
||||
plugins_info_dict.update(d)
|
||||
|
||||
return plugins_info_dict
|
||||
|
||||
def get_plugin_info(self, plugin_name):
|
||||
""" This method is intended to provide information about plugins within
|
||||
a given module's implementation of Base.gen_xml. The return value is a
|
||||
dictionary with data obtained directly from a running Jenkins instance.
|
||||
This allows module authors to differentiate generated XML output based
|
||||
on information such as specific plugin versions.
|
||||
|
||||
:arg string plugin_name: Either the shortName or longName of a plugin
|
||||
as see in a query that looks like:
|
||||
``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2``
|
||||
|
||||
During a 'test' run, it is possible to override JJB's query to a live
|
||||
Jenkins instance by passing it a path to a file containing a YAML list
|
||||
of dictionaries that mimics the plugin properties you want your test
|
||||
output to reflect::
|
||||
|
||||
jenkins-jobs test -p /path/to/plugins-info.yaml
|
||||
|
||||
Below is example YAML that might be included in
|
||||
/path/to/plugins-info.yaml.
|
||||
|
||||
.. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
|
||||
|
||||
"""
|
||||
return self.plugins_dict.get(plugin_name, {})
|
||||
|
||||
def registerHandler(self, category, name, method):
|
||||
cat_dict = self.handlers.get(category, {})
|
||||
if not cat_dict:
|
||||
self.handlers[category] = cat_dict
|
||||
cat_dict[name] = method
|
||||
|
||||
def getHandler(self, category, name):
|
||||
return self.handlers[category][name]
|
||||
|
||||
def dispatch(self, component_type,
|
||||
parser, xml_parent,
|
||||
component, template_data={}):
|
||||
"""This is a method that you can call from your implementation of
|
||||
Base.gen_xml or component. It allows modules to define a type
|
||||
of component, and benefit from extensibility via Python
|
||||
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
|
||||
|
||||
:arg string component_type: the name of the component
|
||||
(e.g., `builder`)
|
||||
:arg YAMLParser parser: the global YAML Parser
|
||||
:arg Element xml_parent: the parent XML element
|
||||
:arg dict template_data: values that should be interpolated into
|
||||
the component definition
|
||||
|
||||
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
|
||||
components of a module.
|
||||
|
||||
See the Publishers module for a simple example of how to use
|
||||
this method.
|
||||
"""
|
||||
|
||||
if component_type not in self.modules_by_component_type:
|
||||
raise JenkinsJobsException("Unknown component type: "
|
||||
"'{0}'.".format(component_type))
|
||||
|
||||
component_list_type = self.modules_by_component_type[component_type] \
|
||||
.component_list_type
|
||||
|
||||
if isinstance(component, dict):
|
||||
# The component is a singleton dictionary of name: dict(args)
|
||||
name, component_data = next(iter(component.items()))
|
||||
if template_data:
|
||||
# Template data contains values that should be interpolated
|
||||
# into the component definition
|
||||
s = yaml.dump(component_data, default_flow_style=False)
|
||||
allow_empty_variables = self.global_config \
|
||||
and self.global_config.has_section('job_builder') \
|
||||
and self.global_config.has_option(
|
||||
'job_builder', 'allow_empty_variables') \
|
||||
and self.global_config.getboolean(
|
||||
'job_builder', 'allow_empty_variables')
|
||||
s = CustomFormatter(
|
||||
allow_empty_variables).format(s, **template_data)
|
||||
component_data = yaml.load(s)
|
||||
else:
|
||||
# The component is a simple string name, eg "run-tests"
|
||||
name = component
|
||||
component_data = {}
|
||||
|
||||
# Look for a component function defined in an entry point
|
||||
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
|
||||
if eps is None:
|
||||
module_eps = list(pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.{0}'.format(component_list_type)))
|
||||
eps = {}
|
||||
for module_ep in module_eps:
|
||||
if module_ep.name in eps:
|
||||
raise JenkinsJobsException(
|
||||
"Duplicate entry point found for component type: "
|
||||
"'{0}', '{0}',"
|
||||
"name: '{1}'".format(component_type, name))
|
||||
eps[module_ep.name] = module_ep
|
||||
|
||||
ModuleRegistry.entry_points_cache[component_list_type] = eps
|
||||
logger.debug("Cached entry point group %s = %s",
|
||||
component_list_type, eps)
|
||||
|
||||
if name in eps:
|
||||
func = eps[name].load()
|
||||
func(parser, xml_parent, component_data)
|
||||
else:
|
||||
# Otherwise, see if it's defined as a macro
|
||||
component = parser.data.get(component_type, {}).get(name)
|
||||
if component:
|
||||
for b in component[component_list_type]:
|
||||
# Pass component_data in as template data to this function
|
||||
# so that if the macro is invoked with arguments,
|
||||
# the arguments are interpolated into the real defn.
|
||||
self.dispatch(component_type,
|
||||
parser, xml_parent, b, component_data)
|
||||
else:
|
||||
raise JenkinsJobsException("Unknown entry point or macro '{0}'"
|
||||
" for component type: '{1}'.".
|
||||
format(name, component_type))
|
||||
|
||||
|
||||
class XmlJob(object):
|
||||
def __init__(self, xml, name):
|
||||
self.xml = xml
|
||||
self.name = name
|
||||
|
||||
def md5(self):
|
||||
return hashlib.md5(self.output()).hexdigest()
|
||||
|
||||
def output(self):
|
||||
out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8'))
|
||||
return out.toprettyxml(indent=' ', encoding='utf-8')
|
||||
|
||||
|
||||
class CacheStorage(object):
|
||||
# ensure each instance of the class has a reference to the required
|
||||
# modules so that they are available to be used when the destructor
|
||||
|
||||
18
jenkins_jobs/constants.py
Normal file
18
jenkins_jobs/constants.py
Normal file
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2015 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Manage JJB Constants
|
||||
|
||||
MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
|
||||
86
jenkins_jobs/formatter.py
Normal file
86
jenkins_jobs/formatter.py
Normal file
@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2015 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Manage interpolation of JJB variables into template strings.
|
||||
|
||||
import logging
|
||||
from pprint import pformat
|
||||
import re
|
||||
from string import Formatter
|
||||
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def deep_format(obj, paramdict, allow_empty=False):
|
||||
"""Apply the paramdict via str.format() to all string objects found within
|
||||
the supplied obj. Lists and dicts are traversed recursively."""
|
||||
# YAML serialisation was originally used to achieve this, but that places
|
||||
# limitations on the values in paramdict - the post-format result must
|
||||
# still be valid YAML (so substituting-in a string containing quotes, for
|
||||
# example, is problematic).
|
||||
if hasattr(obj, 'format'):
|
||||
try:
|
||||
result = re.match('^{obj:(?P<key>\w+)}$', obj)
|
||||
if result is not None:
|
||||
ret = paramdict[result.group("key")]
|
||||
else:
|
||||
ret = CustomFormatter(allow_empty).format(obj, **paramdict)
|
||||
except KeyError as exc:
|
||||
missing_key = exc.message
|
||||
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
|
||||
missing_key, obj, pformat(paramdict))
|
||||
raise JenkinsJobsException(desc)
|
||||
elif isinstance(obj, list):
|
||||
ret = []
|
||||
for item in obj:
|
||||
ret.append(deep_format(item, paramdict, allow_empty))
|
||||
elif isinstance(obj, dict):
|
||||
ret = {}
|
||||
for item in obj:
|
||||
try:
|
||||
ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
|
||||
deep_format(obj[item], paramdict, allow_empty)
|
||||
except KeyError as exc:
|
||||
missing_key = exc.message
|
||||
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
|
||||
missing_key, obj, pformat(paramdict))
|
||||
raise JenkinsJobsException(desc)
|
||||
else:
|
||||
ret = obj
|
||||
return ret
|
||||
|
||||
|
||||
class CustomFormatter(Formatter):
|
||||
"""
|
||||
Custom formatter to allow non-existing key references when formatting a
|
||||
string
|
||||
"""
|
||||
def __init__(self, allow_empty=False):
|
||||
super(CustomFormatter, self).__init__()
|
||||
self.allow_empty = allow_empty
|
||||
|
||||
def get_value(self, key, args, kwargs):
|
||||
try:
|
||||
return Formatter.get_value(self, key, args, kwargs)
|
||||
except KeyError:
|
||||
if self.allow_empty:
|
||||
logger.debug(
|
||||
'Found uninitialized key %s, replaced with empty string',
|
||||
key
|
||||
)
|
||||
return ''
|
||||
raise
|
||||
322
jenkins_jobs/parser.py
Normal file
322
jenkins_jobs/parser.py
Normal file
@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2015 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Manage JJB yaml feature implementation
|
||||
|
||||
import copy
|
||||
import fnmatch
|
||||
import itertools
|
||||
import logging
|
||||
import pkg_resources
|
||||
|
||||
import jenkins_jobs.local_yaml as local_yaml
|
||||
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
from jenkins_jobs.registry import ModuleRegistry
|
||||
from jenkins_jobs.formatter import deep_format
|
||||
from jenkins_jobs.xml_config import XmlJob
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def matches(what, glob_patterns):
|
||||
"""
|
||||
Checks if the given string, ``what``, matches any of the glob patterns in
|
||||
the iterable, ``glob_patterns``
|
||||
|
||||
:arg str what: String that we want to test if it matches a pattern
|
||||
:arg iterable glob_patterns: glob patterns to match (list, tuple, set,
|
||||
etc.)
|
||||
"""
|
||||
return any(fnmatch.fnmatch(what, glob_pattern)
|
||||
for glob_pattern in glob_patterns)
|
||||
|
||||
|
||||
class YamlParser(object):
|
||||
def __init__(self, config=None, plugins_info=None):
|
||||
self.data = {}
|
||||
self.jobs = []
|
||||
self.xml_jobs = []
|
||||
self.config = config
|
||||
self.registry = ModuleRegistry(self.config, plugins_info)
|
||||
self.path = ["."]
|
||||
if self.config:
|
||||
if config.has_section('job_builder') and \
|
||||
config.has_option('job_builder', 'include_path'):
|
||||
self.path = config.get('job_builder',
|
||||
'include_path').split(':')
|
||||
self.keep_desc = self.get_keep_desc()
|
||||
|
||||
def get_keep_desc(self):
|
||||
keep_desc = False
|
||||
if self.config and self.config.has_section('job_builder') and \
|
||||
self.config.has_option('job_builder', 'keep_descriptions'):
|
||||
keep_desc = self.config.getboolean('job_builder',
|
||||
'keep_descriptions')
|
||||
return keep_desc
|
||||
|
||||
def parse_fp(self, fp):
|
||||
data = local_yaml.load(fp, search_path=self.path)
|
||||
if data:
|
||||
if not isinstance(data, list):
|
||||
raise JenkinsJobsException(
|
||||
"The topmost collection in file '{fname}' must be a list,"
|
||||
" not a {cls}".format(fname=getattr(fp, 'name', fp),
|
||||
cls=type(data)))
|
||||
for item in data:
|
||||
cls, dfn = next(iter(item.items()))
|
||||
group = self.data.get(cls, {})
|
||||
if len(item.items()) > 1:
|
||||
n = None
|
||||
for k, v in item.items():
|
||||
if k == "name":
|
||||
n = v
|
||||
break
|
||||
# Syntax error
|
||||
raise JenkinsJobsException("Syntax error, for item "
|
||||
"named '{0}'. Missing indent?"
|
||||
.format(n))
|
||||
name = dfn['name']
|
||||
if name in group:
|
||||
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
|
||||
"already defined".format(fp.name, name))
|
||||
group[name] = dfn
|
||||
self.data[cls] = group
|
||||
|
||||
def parse(self, fn):
|
||||
with open(fn) as fp:
|
||||
self.parse_fp(fp)
|
||||
|
||||
def _handle_dups(self, message):
|
||||
|
||||
if not (self.config and self.config.has_section('job_builder') and
|
||||
self.config.getboolean('job_builder', 'allow_duplicates')):
|
||||
logger.error(message)
|
||||
raise JenkinsJobsException(message)
|
||||
else:
|
||||
logger.warn(message)
|
||||
|
||||
def getJob(self, name):
|
||||
job = self.data.get('job', {}).get(name, None)
|
||||
if not job:
|
||||
return job
|
||||
return self.applyDefaults(job)
|
||||
|
||||
def getJobGroup(self, name):
|
||||
return self.data.get('job-group', {}).get(name, None)
|
||||
|
||||
def getJobTemplate(self, name):
|
||||
job = self.data.get('job-template', {}).get(name, None)
|
||||
if not job:
|
||||
return job
|
||||
return self.applyDefaults(job)
|
||||
|
||||
def applyDefaults(self, data, override_dict=None):
|
||||
if override_dict is None:
|
||||
override_dict = {}
|
||||
|
||||
whichdefaults = data.get('defaults', 'global')
|
||||
defaults = copy.deepcopy(self.data.get('defaults',
|
||||
{}).get(whichdefaults, {}))
|
||||
if defaults == {} and whichdefaults != 'global':
|
||||
raise JenkinsJobsException("Unknown defaults set: '{0}'"
|
||||
.format(whichdefaults))
|
||||
|
||||
for key in override_dict.keys():
|
||||
if key in defaults.keys():
|
||||
defaults[key] = override_dict[key]
|
||||
|
||||
newdata = {}
|
||||
newdata.update(defaults)
|
||||
newdata.update(data)
|
||||
return newdata
|
||||
|
||||
def formatDescription(self, job):
|
||||
if self.keep_desc:
|
||||
description = job.get("description", None)
|
||||
else:
|
||||
description = job.get("description", '')
|
||||
if description is not None:
|
||||
job["description"] = description + \
|
||||
self.get_managed_string().lstrip()
|
||||
|
||||
def expandYaml(self, jobs_glob=None):
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for module in self.registry.modules:
|
||||
if hasattr(module, 'handle_data'):
|
||||
if module.handle_data(self):
|
||||
changed = True
|
||||
|
||||
for job in self.data.get('job', {}).values():
|
||||
if jobs_glob and not matches(job['name'], jobs_glob):
|
||||
logger.debug("Ignoring job {0}".format(job['name']))
|
||||
continue
|
||||
logger.debug("Expanding job '{0}'".format(job['name']))
|
||||
job = self.applyDefaults(job)
|
||||
self.formatDescription(job)
|
||||
self.jobs.append(job)
|
||||
for project in self.data.get('project', {}).values():
|
||||
logger.debug("Expanding project '{0}'".format(project['name']))
|
||||
# use a set to check for duplicate job references in projects
|
||||
seen = set()
|
||||
for jobspec in project.get('jobs', []):
|
||||
if isinstance(jobspec, dict):
|
||||
# Singleton dict containing dict of job-specific params
|
||||
jobname, jobparams = next(iter(jobspec.items()))
|
||||
if not isinstance(jobparams, dict):
|
||||
jobparams = {}
|
||||
else:
|
||||
jobname = jobspec
|
||||
jobparams = {}
|
||||
job = self.getJob(jobname)
|
||||
if job:
|
||||
# Just naming an existing defined job
|
||||
if jobname in seen:
|
||||
self._handle_dups("Duplicate job '{0}' specified "
|
||||
"for project '{1}'".format(
|
||||
jobname, project['name']))
|
||||
seen.add(jobname)
|
||||
continue
|
||||
# see if it's a job group
|
||||
group = self.getJobGroup(jobname)
|
||||
if group:
|
||||
for group_jobspec in group['jobs']:
|
||||
if isinstance(group_jobspec, dict):
|
||||
group_jobname, group_jobparams = \
|
||||
next(iter(group_jobspec.items()))
|
||||
if not isinstance(group_jobparams, dict):
|
||||
group_jobparams = {}
|
||||
else:
|
||||
group_jobname = group_jobspec
|
||||
group_jobparams = {}
|
||||
job = self.getJob(group_jobname)
|
||||
if job:
|
||||
if group_jobname in seen:
|
||||
self._handle_dups(
|
||||
"Duplicate job '{0}' specified for "
|
||||
"project '{1}'".format(group_jobname,
|
||||
project['name']))
|
||||
seen.add(group_jobname)
|
||||
continue
|
||||
template = self.getJobTemplate(group_jobname)
|
||||
# Allow a group to override parameters set by a project
|
||||
d = {}
|
||||
d.update(project)
|
||||
d.update(jobparams)
|
||||
d.update(group)
|
||||
d.update(group_jobparams)
|
||||
# Except name, since the group's name is not useful
|
||||
d['name'] = project['name']
|
||||
if template:
|
||||
self.expandYamlForTemplateJob(d, template,
|
||||
jobs_glob)
|
||||
continue
|
||||
# see if it's a template
|
||||
template = self.getJobTemplate(jobname)
|
||||
if template:
|
||||
d = {}
|
||||
d.update(project)
|
||||
d.update(jobparams)
|
||||
self.expandYamlForTemplateJob(d, template, jobs_glob)
|
||||
else:
|
||||
raise JenkinsJobsException("Failed to find suitable "
|
||||
"template named '{0}'"
|
||||
.format(jobname))
|
||||
# check for duplicate generated jobs
|
||||
seen = set()
|
||||
# walk the list in reverse so that last definition wins
|
||||
for job in self.jobs[::-1]:
|
||||
if job['name'] in seen:
|
||||
self._handle_dups("Duplicate definitions for job '{0}' "
|
||||
"specified".format(job['name']))
|
||||
self.jobs.remove(job)
|
||||
seen.add(job['name'])
|
||||
|
||||
def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
|
||||
dimensions = []
|
||||
template_name = template['name']
|
||||
# reject keys that are not useful during yaml expansion
|
||||
for k in ['jobs']:
|
||||
project.pop(k)
|
||||
for (k, v) in project.items():
|
||||
tmpk = '{{{0}}}'.format(k)
|
||||
if tmpk not in template_name:
|
||||
logger.debug("Variable %s not in name %s, rejecting from job"
|
||||
" matrix expansion.", tmpk, template_name)
|
||||
continue
|
||||
if type(v) == list:
|
||||
dimensions.append(zip([k] * len(v), v))
|
||||
# XXX somewhat hackish to ensure we actually have a single
|
||||
# pass through the loop
|
||||
if len(dimensions) == 0:
|
||||
dimensions = [(("", ""),)]
|
||||
|
||||
for values in itertools.product(*dimensions):
|
||||
params = copy.deepcopy(project)
|
||||
params = self.applyDefaults(params, template)
|
||||
|
||||
expanded_values = {}
|
||||
for (k, v) in values:
|
||||
if isinstance(v, dict):
|
||||
inner_key = next(iter(v))
|
||||
expanded_values[k] = inner_key
|
||||
expanded_values.update(v[inner_key])
|
||||
else:
|
||||
expanded_values[k] = v
|
||||
|
||||
params.update(expanded_values)
|
||||
params = deep_format(params, params)
|
||||
allow_empty_variables = self.config \
|
||||
and self.config.has_section('job_builder') \
|
||||
and self.config.has_option(
|
||||
'job_builder', 'allow_empty_variables') \
|
||||
and self.config.getboolean(
|
||||
'job_builder', 'allow_empty_variables')
|
||||
expanded = deep_format(template, params, allow_empty_variables)
|
||||
|
||||
job_name = expanded.get('name')
|
||||
if jobs_glob and not matches(job_name, jobs_glob):
|
||||
continue
|
||||
|
||||
self.formatDescription(expanded)
|
||||
self.jobs.append(expanded)
|
||||
|
||||
def get_managed_string(self):
|
||||
# The \n\n is not hard coded, because they get stripped if the
|
||||
# project does not otherwise have a description.
|
||||
return "\n\n" + MAGIC_MANAGE_STRING
|
||||
|
||||
def generateXML(self):
|
||||
for job in self.jobs:
|
||||
self.xml_jobs.append(self.getXMLForJob(job))
|
||||
|
||||
def getXMLForJob(self, data):
|
||||
kind = data.get('project-type', 'freestyle')
|
||||
|
||||
for ep in pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.projects', name=kind):
|
||||
Mod = ep.load()
|
||||
mod = Mod(self.registry)
|
||||
xml = mod.root_xml(data)
|
||||
self.gen_xml(xml, data)
|
||||
job = XmlJob(xml, data['name'])
|
||||
return job
|
||||
|
||||
def gen_xml(self, xml, data):
|
||||
for module in self.registry.modules:
|
||||
if hasattr(module, 'gen_xml'):
|
||||
module.gen_xml(self, xml, data)
|
||||
204
jenkins_jobs/registry.py
Normal file
204
jenkins_jobs/registry.py
Normal file
@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2015 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Manage Jenkins plugin module registry.
|
||||
|
||||
import logging
|
||||
import operator
|
||||
import pkg_resources
|
||||
import re
|
||||
import yaml
|
||||
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
from jenkins_jobs.formatter import CustomFormatter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModuleRegistry(object):
|
||||
entry_points_cache = {}
|
||||
|
||||
def __init__(self, config, plugins_list=None):
|
||||
self.modules = []
|
||||
self.modules_by_component_type = {}
|
||||
self.handlers = {}
|
||||
self.global_config = config
|
||||
|
||||
if plugins_list is None:
|
||||
self.plugins_dict = {}
|
||||
else:
|
||||
self.plugins_dict = self._get_plugins_info_dict(plugins_list)
|
||||
|
||||
for entrypoint in pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.modules'):
|
||||
Mod = entrypoint.load()
|
||||
mod = Mod(self)
|
||||
self.modules.append(mod)
|
||||
self.modules.sort(key=operator.attrgetter('sequence'))
|
||||
if mod.component_type is not None:
|
||||
self.modules_by_component_type[mod.component_type] = mod
|
||||
|
||||
@staticmethod
|
||||
def _get_plugins_info_dict(plugins_list):
|
||||
def mutate_plugin_info(plugin_info):
|
||||
"""
|
||||
We perform mutations on a single member of plugin_info here, then
|
||||
return a dictionary with the longName and shortName of the plugin
|
||||
mapped to its plugin info dictionary.
|
||||
"""
|
||||
version = plugin_info.get('version', '0')
|
||||
plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)',
|
||||
r'\g<1>.preview', version)
|
||||
|
||||
aliases = []
|
||||
for key in ['longName', 'shortName']:
|
||||
value = plugin_info.get(key, None)
|
||||
if value is not None:
|
||||
aliases.append(value)
|
||||
|
||||
plugin_info_dict = {}
|
||||
for name in aliases:
|
||||
plugin_info_dict[name] = plugin_info
|
||||
|
||||
return plugin_info_dict
|
||||
|
||||
list_of_dicts = [mutate_plugin_info(v) for v in plugins_list]
|
||||
|
||||
plugins_info_dict = {}
|
||||
for d in list_of_dicts:
|
||||
plugins_info_dict.update(d)
|
||||
|
||||
return plugins_info_dict
|
||||
|
||||
def get_plugin_info(self, plugin_name):
|
||||
""" This method is intended to provide information about plugins within
|
||||
a given module's implementation of Base.gen_xml. The return value is a
|
||||
dictionary with data obtained directly from a running Jenkins instance.
|
||||
This allows module authors to differentiate generated XML output based
|
||||
on information such as specific plugin versions.
|
||||
|
||||
:arg string plugin_name: Either the shortName or longName of a plugin
|
||||
as see in a query that looks like:
|
||||
``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2``
|
||||
|
||||
During a 'test' run, it is possible to override JJB's query to a live
|
||||
Jenkins instance by passing it a path to a file containing a YAML list
|
||||
of dictionaries that mimics the plugin properties you want your test
|
||||
output to reflect::
|
||||
|
||||
jenkins-jobs test -p /path/to/plugins-info.yaml
|
||||
|
||||
Below is example YAML that might be included in
|
||||
/path/to/plugins-info.yaml.
|
||||
|
||||
.. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
|
||||
|
||||
"""
|
||||
return self.plugins_dict.get(plugin_name, {})
|
||||
|
||||
def registerHandler(self, category, name, method):
|
||||
cat_dict = self.handlers.get(category, {})
|
||||
if not cat_dict:
|
||||
self.handlers[category] = cat_dict
|
||||
cat_dict[name] = method
|
||||
|
||||
def getHandler(self, category, name):
|
||||
return self.handlers[category][name]
|
||||
|
||||
def dispatch(self, component_type,
|
||||
parser, xml_parent,
|
||||
component, template_data={}):
|
||||
"""This is a method that you can call from your implementation of
|
||||
Base.gen_xml or component. It allows modules to define a type
|
||||
of component, and benefit from extensibility via Python
|
||||
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
|
||||
|
||||
:arg string component_type: the name of the component
|
||||
(e.g., `builder`)
|
||||
:arg YAMLParser parser: the global YAML Parser
|
||||
:arg Element xml_parent: the parent XML element
|
||||
:arg dict template_data: values that should be interpolated into
|
||||
the component definition
|
||||
|
||||
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
|
||||
components of a module.
|
||||
|
||||
See the Publishers module for a simple example of how to use
|
||||
this method.
|
||||
"""
|
||||
|
||||
if component_type not in self.modules_by_component_type:
|
||||
raise JenkinsJobsException("Unknown component type: "
|
||||
"'{0}'.".format(component_type))
|
||||
|
||||
component_list_type = self.modules_by_component_type[component_type] \
|
||||
.component_list_type
|
||||
|
||||
if isinstance(component, dict):
|
||||
# The component is a singleton dictionary of name: dict(args)
|
||||
name, component_data = next(iter(component.items()))
|
||||
if template_data:
|
||||
# Template data contains values that should be interpolated
|
||||
# into the component definition
|
||||
s = yaml.dump(component_data, default_flow_style=False)
|
||||
allow_empty_variables = self.global_config \
|
||||
and self.global_config.has_section('job_builder') \
|
||||
and self.global_config.has_option(
|
||||
'job_builder', 'allow_empty_variables') \
|
||||
and self.global_config.getboolean(
|
||||
'job_builder', 'allow_empty_variables')
|
||||
s = CustomFormatter(
|
||||
allow_empty_variables).format(s, **template_data)
|
||||
component_data = yaml.load(s)
|
||||
else:
|
||||
# The component is a simple string name, eg "run-tests"
|
||||
name = component
|
||||
component_data = {}
|
||||
|
||||
# Look for a component function defined in an entry point
|
||||
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
|
||||
if eps is None:
|
||||
module_eps = list(pkg_resources.iter_entry_points(
|
||||
group='jenkins_jobs.{0}'.format(component_list_type)))
|
||||
eps = {}
|
||||
for module_ep in module_eps:
|
||||
if module_ep.name in eps:
|
||||
raise JenkinsJobsException(
|
||||
"Duplicate entry point found for component type: "
|
||||
"'{0}', '{0}',"
|
||||
"name: '{1}'".format(component_type, name))
|
||||
eps[module_ep.name] = module_ep
|
||||
|
||||
ModuleRegistry.entry_points_cache[component_list_type] = eps
|
||||
logger.debug("Cached entry point group %s = %s",
|
||||
component_list_type, eps)
|
||||
|
||||
if name in eps:
|
||||
func = eps[name].load()
|
||||
func(parser, xml_parent, component_data)
|
||||
else:
|
||||
# Otherwise, see if it's defined as a macro
|
||||
component = parser.data.get(component_type, {}).get(name)
|
||||
if component:
|
||||
for b in component[component_list_type]:
|
||||
# Pass component_data in as template data to this function
|
||||
# so that if the macro is invoked with arguments,
|
||||
# the arguments are interpolated into the real defn.
|
||||
self.dispatch(component_type,
|
||||
parser, xml_parent, b, component_data)
|
||||
else:
|
||||
raise JenkinsJobsException("Unknown entry point or macro '{0}'"
|
||||
" for component type: '{1}'.".
|
||||
format(name, component_type))
|
||||
33
jenkins_jobs/xml_config.py
Normal file
33
jenkins_jobs/xml_config.py
Normal file
@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2015 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Manage Jenkins XML config file output.
|
||||
|
||||
import hashlib
|
||||
from xml.dom import minidom
|
||||
import xml.etree.ElementTree as XML
|
||||
|
||||
|
||||
class XmlJob(object):
|
||||
def __init__(self, xml, name):
|
||||
self.xml = xml
|
||||
self.name = name
|
||||
|
||||
def md5(self):
|
||||
return hashlib.md5(self.output()).hexdigest()
|
||||
|
||||
def output(self):
|
||||
out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8'))
|
||||
return out.toprettyxml(indent=' ', encoding='utf-8')
|
||||
@ -37,7 +37,8 @@ try:
|
||||
except ImportError:
|
||||
import mock # noqa
|
||||
import jenkins_jobs.local_yaml as yaml
|
||||
from jenkins_jobs.builder import XmlJob, YamlParser
|
||||
from jenkins_jobs.parser import YamlParser
|
||||
from jenkins_jobs.xml_config import XmlJob
|
||||
from jenkins_jobs.modules import (project_flow,
|
||||
project_matrix,
|
||||
project_maven,
|
||||
|
||||
@ -190,7 +190,7 @@ class TestTests(CmdTestsBase):
|
||||
"http://test-jenkins.with.non.default.url:8080/")
|
||||
|
||||
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
|
||||
@mock.patch('jenkins_jobs.builder.ModuleRegistry')
|
||||
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
|
||||
def test_plugins_info_stub_option(self, registry_mock, generateXML_mock):
|
||||
"""
|
||||
Test handling of plugins_info stub option.
|
||||
@ -214,7 +214,7 @@ class TestTests(CmdTestsBase):
|
||||
registry_mock.assert_called_with(self.config, plugins_info_list)
|
||||
|
||||
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
|
||||
@mock.patch('jenkins_jobs.builder.ModuleRegistry')
|
||||
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
|
||||
def test_bogus_plugins_info_stub_option(self, registry_mock,
|
||||
generateXML_mock):
|
||||
"""
|
||||
|
||||
@ -5,7 +5,7 @@ from testscenarios.testcase import TestWithScenarios
|
||||
from six.moves import configparser, StringIO
|
||||
|
||||
from jenkins_jobs import cmd
|
||||
from jenkins_jobs.builder import ModuleRegistry
|
||||
from jenkins_jobs.registry import ModuleRegistry
|
||||
|
||||
|
||||
class ModuleRegistryPluginInfoTestsWithScenarios(TestWithScenarios,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user