jjb/jenkins-job-builder
Code Issues Proposed changes

Add source location and context to error messages

Browse Source 
Change-Id: I2e955c01b71a195bb6ff8ba2bb6f3a64cb3e1f58
This commit is contained in:
Vsevolod Fedorov 2023-04-04 11:48:09 +03:00
parent 5ebd23af38
commit 60e8395c62
158 changed files with 2212 additions and 453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
jenkins_jobs/cached_property.py Normal file
View File

@ -0,0 +1,9 @@
from functools import lru_cache
# cached_property was introduced in Python 3.8.
# TODO: Remove this file when support for Python 3.7 is dropped.
# Recipe from https://stackoverflow.com/a/19979379
def cached_property(fn):
return property(lru_cache()(fn))

9
jenkins_jobs/cli/entry.py
View File

@ -22,6 +22,7 @@ from pathlib import Path
from stevedore import extension from stevedore import extension
import yaml import yaml
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.cli.parser import create_parser from jenkins_jobs.cli.parser import create_parser
from jenkins_jobs.config import JJBConfig from jenkins_jobs.config import JJBConfig
from jenkins_jobs import utils from jenkins_jobs import utils
@ -174,7 +175,13 @@ def main():
argv = sys.argv[1:] argv = sys.argv[1:]
jjb = JenkinsJobs(argv) jjb = JenkinsJobs(argv)
jjb.execute() try:
jjb.execute()
except JenkinsJobsException as x:
print(file=sys.stderr)
for line in x.lines:
print(line, file=sys.stderr)
sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":

2
jenkins_jobs/cli/subcommand/base.py
View File

@ -43,7 +43,7 @@ def matches(name, glob_list):
def filter_matching(item_list, glob_list): def filter_matching(item_list, glob_list):
if not glob_list: if not glob_list:
return item_list return item_list
return [item for item in item_list if matches(item["name"], glob_list)] return [item for item in item_list if matches(item.name, glob_list)]
class BaseSubCommand(metaclass=abc.ABCMeta): class BaseSubCommand(metaclass=abc.ABCMeta):

4
jenkins_jobs/cli/subcommand/delete.py
View File

@ -61,8 +61,8 @@ class DeleteSubCommand(base.JobsSubCommand):
roots = self.load_roots(jjb_config, options.path) roots = self.load_roots(jjb_config, options.path)
jobs = base.filter_matching(roots.generate_jobs(), options.name) jobs = base.filter_matching(roots.generate_jobs(), options.name)
views = base.filter_matching(roots.generate_views(), options.name) views = base.filter_matching(roots.generate_views(), options.name)
job_names = [j["name"] for j in jobs] job_names = [j.name for j in jobs]
view_names = [v["name"] for v in views] view_names = [v.name for v in views]
else: else:
job_names = options.name job_names = options.name
view_names = options.name view_names = options.name

2
jenkins_jobs/cli/subcommand/list.py
View File

@ -49,7 +49,7 @@ class ListSubCommand(base.JobsSubCommand):
if path_list: if path_list:
roots = self.load_roots(jjb_config, path_list) roots = self.load_roots(jjb_config, path_list)
jobs = base.filter_matching(roots.generate_jobs(), glob_list) jobs = base.filter_matching(roots.generate_jobs(), glob_list)
job_names = [j["name"] for j in jobs] job_names = [j.name for j in jobs]
else: else:
jenkins = JenkinsManager(jjb_config) jenkins = JenkinsManager(jjb_config)
job_names = [ job_names = [

27
jenkins_jobs/defaults.py
View File

@ -12,6 +12,9 @@
from dataclasses import dataclass from dataclasses import dataclass
from .loc_loader import LocDict
from .position import Pos
job_contents_keys = { job_contents_keys = {
# Same as for macros. # Same as for macros.
@ -154,34 +157,40 @@ view_contents_keys = {
def split_contents_params(data, contents_keys): def split_contents_params(data, contents_keys):
contents = {key: value for key, value in data.items() if key in contents_keys} contents = data.copy_with(
params = {key: value for key, value in data.items() if key not in contents_keys} {key: value for key, value in data.items() if key in contents_keys}
)
params = data.copy_with(
{key: value for key, value in data.items() if key not in contents_keys}
)
return (contents, params) return (contents, params)
@dataclass @dataclass
class Defaults: class Defaults:
name: str name: str
pos: Pos
params: dict params: dict
contents: dict # Values that go to job contents. contents: dict # Values that go to job contents.
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
contents, params = split_contents_params( contents, params = split_contents_params(
d, job_contents_keys | view_contents_keys d, job_contents_keys | view_contents_keys
) )
defaults = cls(name, params, contents) defaults = cls(name, pos, params, contents)
roots.defaults[name] = defaults roots.defaults[name] = defaults
@classmethod @classmethod
def empty(cls): def empty(cls):
return Defaults("empty", params={}, contents={}) return Defaults("empty", pos=None, params={}, contents={})
def merged_with_global(self, global_): def merged_with_global(self, global_):
return Defaults( return Defaults(
name=f"{self.name}-merged-with-global", name=f"{self.name}-merged-with-global",
params={**global_.params, **self.params}, pos=self.pos,
contents={**global_.contents, **self.contents}, params=LocDict.merge(global_.params, self.params),
contents=LocDict.merge(global_.contents, self.contents),
) )

149
jenkins_jobs/dimensions.py
View File

@ -12,78 +12,91 @@
import itertools import itertools
from .errors import JenkinsJobsException from .errors import Context, JenkinsJobsException
from .loc_loader import LocList, LocDict
def merge_dicts(dict_list): def _decode_axis_value(axis, value, key_pos, value_pos):
result = {} if not isinstance(value, (list, LocList)):
for d in dict_list: yield {axis: value}
result.update(d) return
return result for idx, item in enumerate(value):
if not isinstance(item, (dict, LocDict)):
d = LocDict()
class DimensionsExpander: if type(value) is LocList:
def __init__(self, context): d.set_item(axis, item, key_pos, value.value_pos[idx])
self._context = context else:
d[axis] = item
def enum_dimensions_params(self, axes, params, defaults): yield d
if not axes: continue
# No axes - instantiate one job/view. if len(item.items()) != 1:
yield {}
return
dim_values = []
for axis in axes:
try:
value = params[axis]
except KeyError:
try:
value = defaults[axis]
except KeyError:
continue # May be, value would be received from an another axis values.
value = self._decode_axis_value(axis, value)
dim_values.append(value)
for values in itertools.product(*dim_values):
yield merge_dicts(values)
def _decode_axis_value(self, axis, value):
if not isinstance(value, list):
yield {axis: value}
return
for item in value:
if not isinstance(item, dict):
yield {axis: item}
continue
if len(item.items()) != 1:
raise JenkinsJobsException(
f"Invalid parameter {axis!r} definition for template {self._context!r}:"
f" Expected a value or a dict with single element, but got: {item!r}"
)
value, p = next(iter(item.items()))
yield {
axis: value, # Point axis value.
**p, # Point-specific parameters. May override asis value.
}
def is_point_included(self, exclude_list, params):
return not any(self._match_exclude(params, el) for el in exclude_list or [])
def _match_exclude(self, params, exclude):
if not isinstance(exclude, dict):
raise JenkinsJobsException( raise JenkinsJobsException(
f"Template {self._context!r}: Exclude element should be dict, but is: {exclude!r}" f"Expected a value or a dict with single element, but got: {item!r}",
pos=value.value_pos[idx],
ctx=[Context(f"In pamareter {axis!r} definition", key_pos)],
) )
if not exclude: value, p = next(iter(item.items()))
raise JenkinsJobsException( yield LocDict.merge(
f"Template {self._context!r}: Exclude element should be dict, but is empty: {exclude!r}" {axis: value}, # Point axis value.
) p, # Point-specific parameters. May override axis value.
for axis, value in exclude.items(): )
def enum_dimensions_params(axes, params, defaults):
if not axes:
# No axes - instantiate one job/view.
yield {}
return
dim_values = []
for axis in axes:
try:
value, key_pos, value_pos = params.item_with_pos(axis)
except KeyError:
try: try:
v = params[axis] value = defaults[axis]
except KeyError: except KeyError:
raise JenkinsJobsException( continue # May be, value would be received from an another axis values.
f"Template {self._context!r}: Unknown axis {axis!r} for exclude element: {exclude!r}" value = list(_decode_axis_value(axis, value, key_pos, value_pos))
) dim_values.append(value)
if value != v: for values in itertools.product(*dim_values):
return False yield LocDict.merge(*values)
# All required exclude values are matched.
def _match_exclude(params, exclude, pos):
if not isinstance(exclude, dict):
raise JenkinsJobsException(
f"Expected a dict, but got: {exclude!r}",
pos=pos,
)
if not exclude:
raise JenkinsJobsException(
f"Expected a dict, but is empty: {exclude!r}",
pos=pos,
)
for axis, value in exclude.items():
try:
v = params[axis]
except KeyError:
raise JenkinsJobsException(
f"Unknown axis {axis!r}",
pos=pos,
)
if value != v:
return False
# All required exclude values are matched.
return True
def is_point_included(exclude_list, params, key_pos=None):
if not exclude_list:
return True return True
try:
for idx, exclude in enumerate(exclude_list):
if _match_exclude(params, exclude, exclude_list.value_pos[idx]):
return False
except JenkinsJobsException as x:
raise x.with_context(
f"In template exclude list",
pos=key_pos,
)
return True

58
jenkins_jobs/errors.py
View File

@ -1,6 +1,9 @@
"""Exception classes for jenkins_jobs errors""" """Exception classes for jenkins_jobs errors"""
import inspect import inspect
from dataclasses import dataclass
from .position import Pos
def is_sequence(arg): def is_sequence(arg):
@ -9,8 +12,53 @@ def is_sequence(arg):
) )
def context_lines(message, pos):
if not pos:
return [message]
snippet_lines = [line.rstrip() for line in pos.snippet.splitlines()]
return [
f"{pos.path}:{pos.line+1}:{pos.column+1}: {message}",
*snippet_lines,
]
@dataclass
class Context:
message: str
pos: Pos
@property
def lines(self):
return context_lines(self.message, self.pos)
class JenkinsJobsException(Exception): class JenkinsJobsException(Exception):
pass def __init__(self, message, pos=None, ctx=None):
super().__init__(message)
self.pos = pos
self.ctx = ctx or [] # Context list
@property
def message(self):
return self.args[0]
def with_pos(self, pos):
return JenkinsJobsException(self.message, pos, self.ctx)
def with_context(self, message, pos, ctx=None):
return JenkinsJobsException(
self.message, self.pos, [*(ctx or []), Context(message, pos), *self.ctx]
)
def with_ctx_list(self, ctx):
return JenkinsJobsException(self.message, self.pos, [*ctx, *self.ctx])
@property
def lines(self):
ctx_lines = []
for ctx in self.ctx:
ctx_lines += ctx.lines
return [*ctx_lines, *context_lines(self.message, self.pos)]
class ModuleError(JenkinsJobsException): class ModuleError(JenkinsJobsException):
@ -37,7 +85,7 @@ class ModuleError(JenkinsJobsException):
class InvalidAttributeError(ModuleError): class InvalidAttributeError(ModuleError):
def __init__(self, attribute_name, value, valid_values=None): def __init__(self, attribute_name, value, valid_values=None, pos=None, ctx=None):
message = "'{0}' is an invalid value for attribute {1}.{2}".format( message = "'{0}' is an invalid value for attribute {1}.{2}".format(
value, self.get_module_name(), attribute_name value, self.get_module_name(), attribute_name
) )
@ -47,11 +95,11 @@ class InvalidAttributeError(ModuleError):
", ".join("'{0}'".format(value) for value in valid_values) ", ".join("'{0}'".format(value) for value in valid_values)
) )
super(InvalidAttributeError, self).__init__(message) super().__init__(message, pos, ctx)
class MissingAttributeError(ModuleError): class MissingAttributeError(ModuleError):
def __init__(self, missing_attribute, module_name=None): def __init__(self, missing_attribute, module_name=None, pos=None, ctx=None):
module = module_name or self.get_module_name() module = module_name or self.get_module_name()
if is_sequence(missing_attribute): if is_sequence(missing_attribute):
message = "One of {0} must be present in '{1}'".format( message = "One of {0} must be present in '{1}'".format(
@ -62,7 +110,7 @@ class MissingAttributeError(ModuleError):
missing_attribute, module missing_attribute, module
) )
super(MissingAttributeError, self).__init__(message) super().__init__(message, pos, ctx)
class AttributeConflictError(ModuleError): class AttributeConflictError(ModuleError):

145
jenkins_jobs/expander.py
View File

@ -14,8 +14,9 @@ from functools import partial
from jinja2 import StrictUndefined from jinja2 import StrictUndefined
from .errors import JenkinsJobsException from .errors import Context, JenkinsJobsException
from .formatter import CustomFormatter, enum_str_format_required_params from .formatter import CustomFormatter, enum_str_format_required_params
from .loc_loader import LocDict, LocString, LocList
from .yaml_objects import ( from .yaml_objects import (
J2String, J2String,
J2Yaml, J2Yaml,
@ -27,21 +28,30 @@ from .yaml_objects import (
) )
def expand_dict(expander, obj, params): def expand_dict(expander, obj, params, key_pos, value_pos):
result = {} result = LocDict(pos=obj.pos)
for key, value in obj.items(): for key, value in obj.items():
expanded_key = expander.expand(key, params) expanded_key = expander.expand(key, params, None)
expanded_value = expander.expand(value, params) expanded_value = expander.expand(
result[expanded_key] = expanded_value value, params, obj.key_pos.get(key), obj.value_pos.get(key)
)
result.set_item(
expanded_key, expanded_value, obj.key_pos.get(key), obj.value_pos.get(key)
)
return result return result
def expand_list(expander, obj, params): def expand_list(expander, obj, params, key_pos, value_pos):
return [expander.expand(item, params) for item in obj] items = [
expander.expand(item, params, None, obj.value_pos[idx])
for idx, item in enumerate(obj)
]
value_pos = [obj.value_pos[idx] for idx, _ in enumerate(obj)]
return LocList(items, obj.pos, value_pos)
def expand_tuple(expander, obj, params): def expand_tuple(expander, obj, params, key_pos, value_pos):
return tuple(expander.expand(item, params) for item in obj) return tuple(expander.expand(item, params, None) for item in obj)
class StrExpander: class StrExpander:
@ -49,19 +59,31 @@ class StrExpander:
allow_empty = config.yamlparser["allow_empty_variables"] allow_empty = config.yamlparser["allow_empty_variables"]
self._formatter = CustomFormatter(allow_empty) self._formatter = CustomFormatter(allow_empty)
def __call__(self, obj, params): def __call__(self, obj, params, key_pos, value_pos):
return self._formatter.format(obj, **params) try:
return self._formatter.format(str(obj), **params)
except JenkinsJobsException as x:
lines = str(obj).splitlines()
start_ofs = value_pos.body.index(lines[0])
pre_pad = value_pos.body[:start_ofs]
# Shift position to reflect template position inside yaml file:
if "\n" in pre_pad:
pos = value_pos.with_offset(line_ofs=1)
else:
pos = value_pos.with_offset(column_ofs=start_ofs)
pos = pos.with_contents_start()
raise x.with_pos(pos)
def call_expand(expander, obj, params): def call_expand(expander, obj, params, key_pos, value_pos):
return obj.expand(expander, params) return obj.expand(expander, params)
def call_subst(expander, obj, params): def call_subst(expander, obj, params, key_pos, value_pos):
return obj.subst(expander, params) return obj.subst(expander, params)
def dont_expand(obj, params): def dont_expand(obj, params, key_pos, value_pos):
return obj return obj
@ -90,9 +112,12 @@ class Expander:
} }
self.expanders = { self.expanders = {
dict: partial(expand_dict, self), dict: partial(expand_dict, self),
LocDict: partial(expand_dict, self),
list: partial(expand_list, self), list: partial(expand_list, self),
LocList: partial(expand_list, self),
tuple: partial(expand_tuple, self), tuple: partial(expand_tuple, self),
str: dont_expand, str: dont_expand,
LocString: dont_expand,
bool: dont_expand, bool: dont_expand,
int: dont_expand, int: dont_expand,
float: dont_expand, float: dont_expand,
@ -100,13 +125,15 @@ class Expander:
**_yaml_object_expanders, **_yaml_object_expanders,
} }
def expand(self, obj, params): def expand(self, obj, params, key_pos=None, value_pos=None):
t = type(obj) t = type(obj)
try: try:
expander = self.expanders[t] expander = self.expanders[t]
except KeyError: except KeyError:
raise RuntimeError(f"Do not know how to expand type: {t!r}") raise JenkinsJobsException(
return expander(obj, params) f"Do not know how to expand type: {t!r}", pos=value_pos
)
return expander(obj, params, key_pos, value_pos)
# Expands string formats also. Used in jobs templates and macros with parameters. # Expands string formats also. Used in jobs templates and macros with parameters.
@ -119,27 +146,33 @@ class ParamsExpander(Expander):
self.expanders.update( self.expanders.update(
{ {
str: StrExpander(config), str: StrExpander(config),
LocString: StrExpander(config),
**_yaml_object_expanders, **_yaml_object_expanders,
} }
) )
def call_required_params(obj): def call_required_params(obj, pos):
yield from obj.required_params yield from obj.required_params
def enum_dict_params(obj): def enum_dict_params(obj, pos):
for key, value in obj.items(): for key, value in obj.items():
yield from enum_required_params(key) yield from enum_required_params(key, obj.key_pos.get(key))
yield from enum_required_params(value) yield from enum_required_params(value, obj.value_pos.get(key))
def enum_seq_params(obj): def enum_seq_params(obj, pos):
for value in obj: for idx, value in enumerate(obj):
yield from enum_required_params(value) yield from enum_required_params(value, pos=None)
def no_parameters(obj): def enum_loc_list_params(obj, pos):
for idx, value in enumerate(obj):
yield from enum_required_params(value, obj.value_pos[idx])
def no_parameters(obj, pos):
return [] return []
@ -147,8 +180,11 @@ yaml_classes_enumers = {cls: call_required_params for cls in yaml_classes_list}
param_enumers = { param_enumers = {
str: enum_str_format_required_params, str: enum_str_format_required_params,
LocString: enum_str_format_required_params,
dict: enum_dict_params, dict: enum_dict_params,
LocDict: enum_dict_params,
list: enum_seq_params, list: enum_seq_params,
LocList: enum_loc_list_params,
tuple: enum_seq_params, tuple: enum_seq_params,
bool: no_parameters, bool: no_parameters,
int: no_parameters, int: no_parameters,
@ -161,53 +197,68 @@ param_enumers = {
disable_expand_for = {"template-name"} disable_expand_for = {"template-name"}
def enum_required_params(obj): def enum_required_params(obj, pos):
t = type(obj) t = type(obj)
try: try:
enumer = param_enumers[t] enumer = param_enumers[t]
except KeyError: except KeyError:
raise RuntimeError( raise JenkinsJobsException(
f"Do not know how to enumerate required parameters for type: {t!r}" f"Do not know how to enumerate required parameters for type: {t!r}",
pos=pos,
) )
return enumer(obj) return enumer(obj, pos)
def expand_parameters(expander, param_dict, template_name): def expand_parameters(expander, param_dict):
expanded_params = {} expanded_params = LocDict()
deps = {} # Using dict as ordered set. deps = {} # Variable name -> variable pos.
def deps_context():
return [Context(f"Used by {n}", vp) for n, (kp, vp) in deps.items()]
def expand(name): def expand(name):
try: try:
return expanded_params[name] value = expanded_params[name]
key_pos = expanded_params.key_pos.get(name)
value_pos = expanded_params.value_pos.get(name)
return (value, key_pos, value_pos)
except KeyError: except KeyError:
pass pass
try: try:
format = param_dict[name] format = param_dict[name]
except KeyError: except KeyError:
return StrictUndefined(name=name) return (StrictUndefined(name=name), None, None)
key_pos = param_dict.key_pos.get(name)
value_pos = param_dict.value_pos.get(name)
if name in deps: if name in deps:
raise RuntimeError( expand_ctx = Context(f"While expanding {name!r}", key_pos)
f"While expanding {name!r} for template {template_name!r}:" raise JenkinsJobsException(
f" Recursive parameters usage: {name} <- {' <- '.join(deps)}" f"Recursive parameters usage: {' <- '.join(deps)}",
pos=value_pos,
ctx=[*deps_context(), expand_ctx],
) )
if name in disable_expand_for: if name in disable_expand_for:
value = format value = format
else: else:
required_params = list(enum_required_params(format)) required_params = list(enum_required_params(format, value_pos))
deps[name] = None deps[name] = (key_pos, value_pos)
try: try:
params = {n: expand(n) for n in required_params} params = LocDict()
for n in required_params:
v, kp, vp = expand(n)
params.set_item(n, v, kp, vp)
finally: finally:
deps.popitem() deps.popitem()
try: try:
value = expander.expand(format, params) value = expander.expand(format, params, key_pos, value_pos)
except JenkinsJobsException as x: except JenkinsJobsException as x:
used_by_deps = ", used by".join(f"{d!r}" for d in deps) raise x.with_context(
raise RuntimeError( f"While expanding parameter {name!r}",
f"While expanding {name!r}, used by {used_by_deps}, used by template {template_name!r}: {x}" pos=key_pos,
ctx=deps_context(),
) )
expanded_params[name] = value expanded_params.set_item(name, value, key_pos, value_pos)
return value return (value, key_pos, value_pos)
for name in param_dict: for name in param_dict:
expand(name) expand(name)

11
jenkins_jobs/formatter.py
View File

@ -97,7 +97,7 @@ class CustomFormatter(Formatter):
continue continue
arg_used, rest = _string.formatter_field_name_split(field_name) arg_used, rest = _string.formatter_field_name_split(field_name)
if arg_used == "" or type(arg_used) is int: if arg_used == "" or type(arg_used) is int:
raise RuntimeError( raise JenkinsJobsException(
f"Positional format arguments are not supported: {format_string!r}" f"Positional format arguments are not supported: {format_string!r}"
) )
yield arg_used yield arg_used
@ -121,11 +121,14 @@ class CustomFormatter(Formatter):
raise JenkinsJobsException(f"Missing parameter: {key!r}") raise JenkinsJobsException(f"Missing parameter: {key!r}")
def enum_str_format_required_params(format): def enum_str_format_required_params(format, pos):
formatter = CustomFormatter() formatter = CustomFormatter()
yield from formatter.enum_required_params(format) try:
yield from formatter.enum_required_params(str(format))
except JenkinsJobsException as x:
raise x.with_pos(pos)
def enum_str_format_param_defaults(format): def enum_str_format_param_defaults(format):
formatter = CustomFormatter() formatter = CustomFormatter()
yield from formatter.enum_param_defaults(format) yield from formatter.enum_param_defaults(str(format))

58
jenkins_jobs/job.py
View File

@ -12,6 +12,8 @@
from dataclasses import dataclass from dataclasses import dataclass
from .errors import JenkinsJobsException
from .loc_loader import LocDict
from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group
from .defaults import split_contents_params, job_contents_keys from .defaults import split_contents_params, job_contents_keys
@ -22,15 +24,15 @@ class JobBase(RootBase):
folder: str folder: str
@classmethod @classmethod
def from_dict(cls, config, roots, expander, data): def from_dict(cls, config, roots, expander, data, pos):
keep_descriptions = config.yamlparser["keep_descriptions"] keep_descriptions = config.yamlparser["keep_descriptions"]
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
id = d.pop("id", None) id = d.pop_loc_string("id", None)
description = d.pop("description", None) description = d.pop_loc_string("description", None)
defaults = d.pop("defaults", "global") defaults = d.pop_loc_string("defaults", "global")
project_type = d.pop("project-type", None) project_type = d.pop_loc_string("project-type", None)
folder = d.pop("folder", None) folder = d.pop_loc_string("folder", None)
contents, params = split_contents_params(d, job_contents_keys) contents, params = split_contents_params(d, job_contents_keys)
return cls( return cls(
roots.defaults, roots.defaults,
@ -38,6 +40,7 @@ class JobBase(RootBase):
keep_descriptions, keep_descriptions,
id, id,
name, name,
pos,
description, description,
defaults, defaults,
params, params,
@ -47,10 +50,10 @@ class JobBase(RootBase):
) )
def _as_dict(self): def _as_dict(self):
data = { data = LocDict.merge(
"name": self._full_name, {"name": self._full_name},
**self.contents, self.contents,
} )
if self.project_type: if self.project_type:
data["project-type"] = self.project_type data["project-type"] = self.project_type
return data return data
@ -65,17 +68,23 @@ class JobBase(RootBase):
class Job(JobBase, NonTemplateRootMixin): class Job(JobBase, NonTemplateRootMixin):
@classmethod @classmethod
def add(cls, config, roots, expander, param_expander, data): def add(cls, config, roots, expander, param_expander, data, pos):
job = cls.from_dict(config, roots, expander, data) job = cls.from_dict(config, roots, expander, data, pos)
roots.assign(roots.jobs, job.id, job, "job") roots.assign(roots.jobs, job.id, job, "job")
def __str__(self):
return f"job {self.name!r}"
class JobTemplate(JobBase, TemplateRootMixin): class JobTemplate(JobBase, TemplateRootMixin):
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
template = cls.from_dict(config, roots, params_expander, data) template = cls.from_dict(config, roots, params_expander, data, pos)
roots.assign(roots.job_templates, template.id, template, "job template") roots.assign(roots.job_templates, template.id, template, "job template")
def __str__(self):
return f"job template {self.name!r}"
@dataclass @dataclass
class JobGroup(Group): class JobGroup(Group):
@ -83,15 +92,16 @@ class JobGroup(Group):
_job_templates: dict _job_templates: dict
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
job_specs = [ try:
cls._spec_from_dict(item, error_context=f"Job group {name}") job_specs = cls._specs_from_list(d.pop("jobs", None))
for item in d.pop("jobs", []) except JenkinsJobsException as x:
] raise x.with_context(f"In job {name!r}", pos=pos)
group = cls( group = cls(
name, name,
pos,
job_specs, job_specs,
d, d,
roots.jobs, roots.jobs,
@ -100,7 +110,7 @@ class JobGroup(Group):
roots.assign(roots.job_groups, group.name, group, "job group") roots.assign(roots.job_groups, group.name, group, "job group")
def __str__(self): def __str__(self):
return f"Job group {self.name}" return f"job group {self.name!r}"
@property @property
def _root_dicts(self): def _root_dicts(self):

34
jenkins_jobs/loader.py
View File

@ -14,9 +14,8 @@ import io
import logging import logging
from functools import partial from functools import partial
import yaml
from .errors import JenkinsJobsException from .errors import JenkinsJobsException
from .loc_loader import LocLoader
from .yaml_objects import BaseYamlObject from .yaml_objects import BaseYamlObject
from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list
from .roots import root_adders from .roots import root_adders
@ -24,13 +23,13 @@ from .roots import root_adders
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Loader(yaml.Loader): class Loader(LocLoader):
@classmethod @classmethod
def empty(cls, jjb_config): def empty(cls, jjb_config):
return cls(io.StringIO(), jjb_config) return cls(io.StringIO(), jjb_config)
def __init__(self, stream, jjb_config, source_path=None, anchors=None): def __init__(self, stream, jjb_config, source_path=None, anchors=None):
super().__init__(stream) super().__init__(stream, source_path)
self.jjb_config = jjb_config self.jjb_config = jjb_config
self.source_path = source_path self.source_path = source_path
self._retain_anchors = jjb_config.yamlparser["retain_anchors"] self._retain_anchors = jjb_config.yamlparser["retain_anchors"]
@ -74,10 +73,10 @@ def load_deprecated_yaml(tag, cls, loader, node):
for cls in yaml_classes_list: for cls in yaml_classes_list:
yaml.add_constructor(cls.yaml_tag, cls.from_yaml, Loader) Loader.add_constructor(cls.yaml_tag, cls.from_yaml)
for tag, cls in deprecated_yaml_tags: for tag, cls in deprecated_yaml_tags:
yaml.add_constructor(tag, partial(load_deprecated_yaml, tag, cls), Loader) Loader.add_constructor(tag, partial(load_deprecated_yaml, tag, cls))
def is_stdin(path): def is_stdin(path):
@ -122,19 +121,21 @@ def load_files(config, roots, path_list):
data = loader.load_path(path) data = loader.load_path(path)
if not isinstance(data, list): if not isinstance(data, list):
raise JenkinsJobsException( raise JenkinsJobsException(
f"The topmost collection in file '{path}' must be a list," f"The topmost collection must be a list, but is: {data}",
f" not a {type(data)}" pos=data.pos,
) )
for item in data: for idx, item in enumerate(data):
if not isinstance(item, dict): if not isinstance(item, dict):
raise JenkinsJobsException( raise JenkinsJobsException(
f"{path}: Topmost list should contain single-item dict," f"Topmost list should contain single-item dict,"
f" not a {type(item)}. Missing indent?" f" not a {type(item)}. Missing indent?",
pos=data.value_pos[idx],
) )
if len(item) != 1: if len(item) != 1:
raise JenkinsJobsException( raise JenkinsJobsException(
f"{path}: Topmost dict should be single-item," f"Topmost dict should be single-item,"
f" but have keys {item.keys()}. Missing indent?" f" but have keys {list(item.keys())}. Missing indent?",
pos=item.pos,
) )
kind, contents = next(iter(item.items())) kind, contents = next(iter(item.items()))
if kind.startswith("_"): if kind.startswith("_"):
@ -145,7 +146,8 @@ def load_files(config, roots, path_list):
adder = root_adders[kind] adder = root_adders[kind]
except KeyError: except KeyError:
raise JenkinsJobsException( raise JenkinsJobsException(
f"{path}: Unknown topmost element type : {kind!r}," f"Unknown topmost element type : {kind!r};"
f" Known are: {','.join(root_adders)}." f" known are: {','.join(root_adders)}.",
pos=item.pos,
) )
adder(config, roots, expander, params_expander, contents) adder(config, roots, expander, params_expander, contents, item.pos)

161
jenkins_jobs/loc_loader.py Normal file
View File

@ -0,0 +1,161 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import UserString
import yaml
from .errors import JenkinsJobsException
from .position import Pos
class LocDict(dict):
"""dict implementation with added source position information"""
def __init__(self, value=None, pos=None, key_pos=None, value_pos=None):
super().__init__(value or [])
self.pos = pos
self.key_pos = key_pos or {} # key -> key pos.
self.value_pos = value_pos or {} # key -> value pos.
def item_with_pos(self, key):
value = self[key] # KeyError is propagated from here.
key_pos = self.key_pos.get(key)
value_pos = self.value_pos.get(key)
return (value, key_pos, value_pos)
def pop_loc_string(self, key, default_value):
value = super().pop(key, default_value)
if type(value) is str:
return LocString(value, self.value_pos.get(key))
else:
return value
def pop_required_loc_string(self, name):
try:
value = self.pop(name)
except KeyError:
raise JenkinsJobsException(
f"Missing required element: {name!r}",
pos=self.pos,
)
return LocString(value, self.value_pos.get(name))
def pop_required_element(self, name):
try:
return self.pop(name)
except KeyError:
raise JenkinsJobsException(
f"Missing required element: {name!r}",
pos=self.pos,
)
def copy(self):
return LocDict(self, self.pos, self.key_pos, self.value_pos)
def copy_with(self, value):
return LocDict(value, self.pos, self.key_pos, self.value_pos)
def __setitem__(self, key, value):
if type(value) is LocString:
super().__setitem__(key, str(value))
self.value_pos[key] = value.pos
else:
super().__setitem__(key, value)
def set_item(self, key, value, key_pos, value_pos):
self[key] = value
if key_pos:
self.key_pos[key] = key_pos
if value_pos:
self.value_pos[key] = value_pos
@classmethod
def merge(cls, *args, pos=None):
result = LocDict(pos=pos)
for d in args:
result.update(d)
if type(d) is cls:
result.key_pos.update(d.key_pos)
result.value_pos.update(d.value_pos)
return result
def update(self, d):
super().update(d)
if type(d) is LocDict:
self.key_pos.update(d.key_pos)
self.value_pos.update(d.value_pos)
class LocList(list):
"""list implementation with added source position information"""
def __init__(self, value=None, pos=None, value_pos=None):
if value is None:
value = []
super().__init__(value)
self.pos = pos
self.value_pos = value_pos or [None for _ in value] # Value pos list.
def copy(self):
return LocList(self, self.pos, self.value_pos)
class LocString(UserString):
"""str implementation with added source position information"""
def __init__(self, value="", pos=None):
super().__init__(value)
self.pos = pos
class LocLoader(yaml.Loader):
"""Load YAML and store source position information"""
def __init__(self, stream, file_path, line_ofs=0, column_ofs=0):
super().__init__(stream)
if file_path:
# Override one set by yaml Reader. Used to construct marks.
self.name = file_path
self._line_ofs = line_ofs
self._column_ofs = column_ofs
def pos_from_node(self, node):
return Pos.from_node(node, self._line_ofs, self._column_ofs)
def construct_yaml_map(self, node):
data = LocDict(pos=self.pos_from_node(node))
yield data
value = self.construct_mapping(node)
data.update(value)
data.key_pos.update(
{
key_node.value: self.pos_from_node(key_node)
for key_node, value_node in node.value
}
)
data.value_pos.update(
{
key_node.value: self.pos_from_node(value_node)
for key_node, value_node in node.value
}
)
def construct_yaml_seq(self, node):
data = LocList(pos=self.pos_from_node(node))
yield data
data.extend(self.construct_sequence(node))
data.value_pos.extend(self.pos_from_node(item_node) for item_node in node.value)
LocLoader.add_constructor("tag:yaml.org,2002:map", LocLoader.construct_yaml_map)
LocLoader.add_constructor("tag:yaml.org,2002:seq", LocLoader.construct_yaml_seq)

24
jenkins_jobs/macro.py
View File

@ -14,6 +14,7 @@ from dataclasses import dataclass
from functools import partial from functools import partial
from .errors import JenkinsJobsException from .errors import JenkinsJobsException
from .position import Pos
macro_specs = [ macro_specs = [
@ -33,20 +34,31 @@ macro_specs = [
@dataclass @dataclass
class Macro: class Macro:
name: str name: str
pos: Pos
elements: list elements: list
@classmethod @classmethod
def add( def add(
cls, type_name, elements_name, config, roots, expander, params_expander, data cls,
type_name,
elements_name,
config,
roots,
expander,
params_expander,
data,
pos,
): ):
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
elements = d.pop(elements_name) elements = d.pop_required_element(elements_name)
if d: if d:
example_key = next(iter(d.keys()))
raise JenkinsJobsException( raise JenkinsJobsException(
f"Macro {type_name} {name!r}: unexpected elements: {','.join(d.keys())}" f"In {type_name} macro {name!r}: unexpected elements: {','.join(d.keys())}",
pos=data.key_pos.get(example_key),
) )
macro = cls(name, elements or []) macro = cls(name, pos, elements or [])
roots.assign(roots.macros[type_name], name, macro, "macro") roots.assign(roots.macros[type_name], name, macro, "macro")

21
jenkins_jobs/modules/project_multibranch.py
View File

@ -87,7 +87,7 @@ import six
from jenkins_jobs.modules.scm import git_extensions from jenkins_jobs.modules.scm import git_extensions
from jenkins_jobs.errors import InvalidAttributeError, MissingAttributeError from jenkins_jobs.errors import InvalidAttributeError, MissingAttributeError
from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.errors import Context, JenkinsJobsException
from jenkins_jobs.xml_config import remove_ignorable_whitespace from jenkins_jobs.xml_config import remove_ignorable_whitespace
logger = logging.getLogger(str(__name__)) logger = logging.getLogger(str(__name__))
@ -1838,14 +1838,24 @@ def apply_property_strategies(props_elem, props_list):
) )
if isinstance(tbopc_val, dict): if isinstance(tbopc_val, dict):
if "comment" not in tbopc_val: if "comment" not in tbopc_val:
raise MissingAttributeError("trigger-build-on-pr-comment[comment]") raise MissingAttributeError(
"trigger-build-on-pr-comment[comment]",
pos=dbs_list.key_pos.get("trigger-build-on-pr-comment"),
)
XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val["comment"] XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val["comment"]
if tbopc_val.get("allow-untrusted-users", False): if tbopc_val.get("allow-untrusted-users", False):
XML.SubElement(tbopc_elem, "allowUntrusted").text = "true" XML.SubElement(tbopc_elem, "allowUntrusted").text = "true"
elif isinstance(tbopc_val, str): elif isinstance(tbopc_val, str):
XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val XML.SubElement(tbopc_elem, "commentBody").text = tbopc_val
else: else:
raise InvalidAttributeError("trigger-build-on-pr-comment", tbopc_val) attr = "trigger-build-on-pr-comment"
ctx = [Context(f"For attribute {attr!r}", dbs_list.key_pos.get(attr))]
raise InvalidAttributeError(
attr,
tbopc_val,
pos=dbs_list.value_pos.get(attr),
ctx=ctx,
)
for opt in pcb_bool_opts: for opt in pcb_bool_opts:
opt_value = dbs_list.get(opt, None) opt_value = dbs_list.get(opt, None)
if opt_value: if opt_value:
@ -1861,7 +1871,10 @@ def apply_property_strategies(props_elem, props_list):
# no sub-elements in this case # no sub-elements in this case
pass pass
else: else:
raise InvalidAttributeError(opt, opt_value) ctx = Context(f"For attribute {opt!r}", dbs_list.key_pos.get(opt))
raise InvalidAttributeError(
opt, opt_value, pos=dbs_list.value_pos.get(opt), ctx=[ctx]
)
def add_filter_branch_pr_behaviors(traits, data): def add_filter_branch_pr_behaviors(traits, data):

4
jenkins_jobs/modules/view_nested.py
View File

@ -42,6 +42,7 @@ Example:
import xml.etree.ElementTree as XML import xml.etree.ElementTree as XML
import jenkins_jobs.modules.base import jenkins_jobs.modules.base
import jenkins_jobs.modules.helpers as helpers import jenkins_jobs.modules.helpers as helpers
from jenkins_jobs.root_base import JobViewData
from jenkins_jobs.xml_config import XmlViewGenerator from jenkins_jobs.xml_config import XmlViewGenerator
COLUMN_DICT = { COLUMN_DICT = {
@ -66,9 +67,10 @@ class Nested(jenkins_jobs.modules.base.Base):
v_xml = XML.SubElement(root, "views") v_xml = XML.SubElement(root, "views")
views = data.get("views", []) views = data.get("views", [])
view_data_list = [JobViewData(v) for v in views]
xml_view_generator = XmlViewGenerator(self.registry) xml_view_generator = XmlViewGenerator(self.registry)
xml_views = xml_view_generator.generateXML(views) xml_views = xml_view_generator.generateXML(view_data_list)
for xml_job in xml_views: for xml_job in xml_views:
v_xml.append(xml_job.xml) v_xml.append(xml_job.xml)

2
jenkins_jobs/modules/wrappers.py
View File

@ -1828,7 +1828,7 @@ def pre_scm_buildstep(registry, xml_parent, data):
xml_parent, "org.jenkinsci.plugins.preSCMbuildstep." "PreSCMBuildStepsWrapper" xml_parent, "org.jenkinsci.plugins.preSCMbuildstep." "PreSCMBuildStepsWrapper"
) )
bs = XML.SubElement(bsp, "buildSteps") bs = XML.SubElement(bsp, "buildSteps")
stepList = data if type(data) is list else data.get("buildsteps") stepList = data if isinstance(data, list) else data.get("buildsteps")
for step in stepList: for step in stepList:
for edited_node in create_builders(registry, step): for edited_node in create_builders(registry, step):

93
jenkins_jobs/position.py Normal file
View File

@ -0,0 +1,93 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import yaml
if sys.version_info >= (3, 8):
from functools import cached_property
else:
from .cached_property import cached_property
LINE_SEPARATORS = "\0\r\n\x85\u2028\u2029"
WHITESPACE_CHARS = " \t"
class Pos:
@classmethod
def from_node(cls, node, line_ofs=0, column_ofs=0):
mark = node.start_mark
return cls(mark, mark.name, mark.line + line_ofs, mark.column + column_ofs)
@classmethod
def from_file(cls, path, text):
mark = yaml.Mark(str(path), 0, 0, 0, text, 0)
return cls(mark, path, 0, 0)
def __init__(self, mark, path, line, column):
self._mark = mark
self.path = path
self.line = line # Starts from 0.
self.column = column # Starts from 0.
def __repr__(self):
return f"<Pos {self.path}:{self.line}:{self.column}>"
def with_offset(self, line_ofs=0, column_ofs=0):
line_ptr = self._move_ptr_by_lines(line_ofs)
ptr = line_ptr + column_ofs
mark = self._clone_mark(self._mark, ptr)
if line_ofs:
column = column_ofs # Start from new line.
else:
column = self.column + column_ofs
return Pos(mark, self.path, self.line + line_ofs, column)
def with_contents_start(self):
ptr = self._mark.pointer
buf = self._mark.buffer
while (
ptr < len(buf)
and buf[ptr] not in LINE_SEPARATORS
and buf[ptr] in WHITESPACE_CHARS
):
ptr += 1
mark = self._clone_mark(self._mark, ptr)
return Pos(mark, self.path, self.line, self.column + ptr - self._mark.pointer)
@cached_property
def snippet(self):
return self._mark.get_snippet(max_length=100)
@cached_property
def body(self):
return self._mark.buffer[self._mark.pointer :]
def _clone_mark(self, mark, ptr):
return yaml.Mark(
mark.name,
mark.index,
mark.line,
mark.column,
mark.buffer,
ptr,
)
def _move_ptr_by_lines(self, line_ofs):
ptr = self._mark.pointer
buf = self._mark.buffer
while line_ofs > 0 and ptr < len(buf):
if buf[ptr] in LINE_SEPARATORS:
line_ofs -= 1
ptr += 1
return ptr

27
jenkins_jobs/project.py
View File

@ -12,6 +12,8 @@
from dataclasses import dataclass from dataclasses import dataclass
from .errors import JenkinsJobsException
from .position import Pos
from .root_base import GroupBase from .root_base import GroupBase
@ -24,24 +26,22 @@ class Project(GroupBase):
_view_templates: dict _view_templates: dict
_view_groups: dict _view_groups: dict
name: str name: str
pos: Pos
defaults_name: str defaults_name: str
job_specs: list # list[Spec] job_specs: list # list[Spec]
view_specs: list # list[Spec] view_specs: list # list[Spec]
params: dict params: dict
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
defaults = d.pop("defaults", None) defaults = d.pop_loc_string("defaults", None)
job_specs = [ try:
cls._spec_from_dict(item, error_context=f"Project {name}") job_specs = cls._specs_from_list(d.pop("jobs", None))
for item in d.pop("jobs", []) view_specs = cls._specs_from_list(d.pop("views", None))
] except JenkinsJobsException as x:
view_specs = [ raise x.with_context(f"In project {name!r}", pos=pos)
cls._spec_from_dict(item, error_context=f"Project {name}")
for item in d.pop("views", [])
]
project = cls( project = cls(
roots.jobs, roots.jobs,
roots.job_templates, roots.job_templates,
@ -50,6 +50,7 @@ class Project(GroupBase):
roots.view_templates, roots.view_templates,
roots.view_groups, roots.view_groups,
name, name,
pos,
defaults, defaults,
job_specs, job_specs,
view_specs, view_specs,
@ -58,7 +59,7 @@ class Project(GroupBase):
roots.assign(roots.projects, project.name, project, "project") roots.assign(roots.projects, project.name, project, "project")
def __str__(self): def __str__(self):
return f"Project {self.name}" return f"project {self.name!r}"
@property @property
def _my_params(self): def _my_params(self):

21
jenkins_jobs/registry.py
View File

@ -172,9 +172,9 @@ class ModuleRegistry(object):
def amend_job_dicts(self, job_data_list): def amend_job_dicts(self, job_data_list):
while True: while True:
changed = False changed = False
for data in job_data_list: for job in job_data_list:
for module in self.modules: for module in self.modules:
if module.amend_job_dict(data): if module.amend_job_dict(job.data):
changed = True changed = True
if not changed: if not changed:
break break
@ -247,9 +247,15 @@ class ModuleRegistry(object):
component_data, component_type, eps, job_data, macro, name, xml_parent component_data, component_type, eps, job_data, macro, name, xml_parent
) )
elif name in eps: elif name in eps:
func = eps[name] try:
kwargs = self._filter_kwargs(func, job_data=job_data) func = eps[name]
func(self, xml_parent, component_data, **kwargs) kwargs = self._filter_kwargs(func, job_data=job_data)
func(self, xml_parent, component_data, **kwargs)
except JenkinsJobsException as x:
raise x.with_context(
f"In {component_type} {name!r}",
pos=component.pos,
)
else: else:
raise JenkinsJobsException( raise JenkinsJobsException(
"Unknown entry point or macro '{0}' " "Unknown entry point or macro '{0}' "
@ -280,7 +286,10 @@ class ModuleRegistry(object):
try: try:
element = expander.expand(b, expander_params) element = expander.expand(b, expander_params)
except JenkinsJobsException as x: except JenkinsJobsException as x:
raise JenkinsJobsException(f"While expanding macro {name!r}: {x}") raise x.with_context(
f"While expanding macro {name!r}",
pos=macro.pos,
)
# Pass component_data in as template data to this function # Pass component_data in as template data to this function
# so that if the macro is invoked with arguments, # so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn. # the arguments are interpolated into the real defn.

203
jenkins_jobs/root_base.py
View File

@ -11,14 +11,32 @@
# under the License. # under the License.
from collections import namedtuple from collections import namedtuple
from dataclasses import dataclass from dataclasses import dataclass, field
from typing import List
from .constants import MAGIC_MANAGE_STRING from .constants import MAGIC_MANAGE_STRING
from .errors import JenkinsJobsException from .errors import Context, JenkinsJobsException
from .loc_loader import LocDict, LocString
from .position import Pos
from .formatter import enum_str_format_required_params, enum_str_format_param_defaults from .formatter import enum_str_format_required_params, enum_str_format_param_defaults
from .expander import Expander, expand_parameters from .expander import Expander, expand_parameters
from .defaults import Defaults from .defaults import Defaults
from .dimensions import DimensionsExpander from .dimensions import enum_dimensions_params, is_point_included
@dataclass
class JobViewData:
"""Expanded job or view data, with added source context. Fed into xml generator"""
data: LocDict
context: List[Context] = field(default_factory=list)
@property
def name(self):
return self.data["name"]
def with_context(self, message, pos):
return JobViewData(self.data, [Context(message, pos), *self.context])
@dataclass @dataclass
@ -30,6 +48,7 @@ class RootBase:
_keep_descriptions: bool _keep_descriptions: bool
_id: str _id: str
name: str name: str
pos: Pos
description: str description: str
defaults_name: str defaults_name: str
params: dict params: dict
@ -42,12 +61,19 @@ class RootBase:
else: else:
return self.name return self.name
@property
def title(self):
return str(self).capitalize()
def _format_description(self, params): def _format_description(self, params):
if self.description is None: if self.description is None:
defaults = self._pick_defaults(self.defaults_name) defaults = self._pick_defaults(self.defaults_name)
description = defaults.params.get("description") description = defaults.params.get("description")
else: else:
description = self.description if type(self.description) is LocString:
description = str(self.description)
else:
description = self.description
if description is None and self._keep_descriptions: if description is None and self._keep_descriptions:
return {} return {}
expanded_desc = self._expander.expand(description, params) expanded_desc = self._expander.expand(description, params)
@ -60,8 +86,9 @@ class RootBase:
if name == "global": if name == "global":
return Defaults.empty() return Defaults.empty()
raise JenkinsJobsException( raise JenkinsJobsException(
f"Job template {self.name!r} wants defaults {self.defaults_name!r}" f"{self.title} wants defaults {self.defaults_name!r}"
" but it was never defined" " but it was never defined",
pos=name.pos,
) )
if name == "global": if name == "global":
return defaults return defaults
@ -73,15 +100,21 @@ class RootBase:
class NonTemplateRootMixin: class NonTemplateRootMixin:
def top_level_generate_items(self): def top_level_generate_items(self):
defaults = self._pick_defaults(self.defaults_name, merge_global=False) try:
description = self._format_description(params={}) defaults = self._pick_defaults(self.defaults_name, merge_global=False)
data = self._as_dict() description = self._format_description(params={})
contents = self._expander.expand(data, self.params) raw_data = self._as_dict()
yield { contents = self._expander.expand(raw_data, self.params)
**defaults.contents, data = LocDict.merge(
**contents, defaults.contents,
**description, contents,
} description,
pos=self.pos,
)
context = [Context(f"In {self}", self.pos)]
yield JobViewData(data, context)
except JenkinsJobsException as x:
raise x.with_context(f"In {self}", pos=self.pos)
def generate_items(self, defaults_name, params): def generate_items(self, defaults_name, params):
# Do not produce jobs/views from under project - they are produced when # Do not produce jobs/views from under project - they are produced when
@ -91,62 +124,77 @@ class NonTemplateRootMixin:
class TemplateRootMixin: class TemplateRootMixin:
def generate_items(self, defaults_name, params): def generate_items(self, defaults_name, params):
defaults = self._pick_defaults(defaults_name or self.defaults_name) try:
item_params = { defaults = self._pick_defaults(defaults_name or self.defaults_name)
**defaults.params, item_params = LocDict.merge(
**self.params, defaults.params,
**params, self.params,
"template-name": self.name, params,
} {"template-name": self.name},
if self._id:
item_params["id"] = self._id
contents = {
**defaults.contents,
**self._as_dict(),
}
axes = list(enum_str_format_required_params(self.name))
axes_defaults = dict(enum_str_format_param_defaults(self.name))
dim_expander = DimensionsExpander(context=self.name)
for dim_params in dim_expander.enum_dimensions_params(
axes, item_params, axes_defaults
):
instance_params = {
**item_params,
**dim_params,
}
expanded_params = expand_parameters(
self._expander, instance_params, template_name=self.name
) )
exclude_list = expanded_params.get("exclude") if self._id:
if not dim_expander.is_point_included(exclude_list, expanded_params): item_params["id"] = self._id
continue contents = LocDict.merge(
description = self._format_description(expanded_params) defaults.contents,
expanded_contents = self._expander.expand(contents, expanded_params) self._as_dict(),
yield { )
**expanded_contents, axes = list(enum_str_format_required_params(self.name, self.name.pos))
**description, axes_defaults = dict(enum_str_format_param_defaults(self.name))
} for dim_params in enum_dimensions_params(axes, item_params, axes_defaults):
instance_params = LocDict.merge(
item_params,
dim_params,
)
expanded_params = expand_parameters(self._expander, instance_params)
if not is_point_included(
exclude_list=expanded_params.get("exclude"),
params=expanded_params,
key_pos=expanded_params.key_pos.get("exclude"),
):
continue
description = self._format_description(expanded_params)
expanded_contents = self._expander.expand(contents, expanded_params)
data = LocDict.merge(
expanded_contents,
description,
pos=self.pos,
)
context = [Context(f"In {self}", self.pos)]
yield JobViewData(data, context)
except JenkinsJobsException as x:
raise x.with_context(f"In {self}", pos=self.pos)
class GroupBase: class GroupBase:
Spec = namedtuple("Spec", "name params") Spec = namedtuple("Spec", "name params pos")
def __repr__(self): def __repr__(self):
return f"<{self}>" return f"<{self}>"
@classmethod @classmethod
def _spec_from_dict(cls, d, error_context): def _specs_from_list(cls, spec_list=None):
if isinstance(d, str): if spec_list is None:
return cls.Spec(d, params={}) return []
return [
cls._spec_from_dict(item, spec_list.value_pos[idx])
for idx, item in enumerate(spec_list)
]
@classmethod
def _spec_from_dict(cls, d, pos):
if isinstance(d, (str, LocString)):
return cls.Spec(d, params={}, pos=pos)
if not isinstance(d, dict): if not isinstance(d, dict):
raise JenkinsJobsException( raise JenkinsJobsException(
f"{error_context}: Job/view spec should name or dict," "Job/view spec should name or dict,"
f" but is {type(d)}. Missing indent?" f" but is {type(d)} ({d!r}). Missing indent?",
pos=pos,
) )
if len(d) != 1: if len(d) != 1:
raise JenkinsJobsException( raise JenkinsJobsException(
f"{error_context}: Job/view dict should be single-item," "Job/view dict should be single-item,"
f" but have keys {list(d.keys())}. Missing indent?" f" but have keys {list(d.keys())}. Missing indent?",
pos=d.pos,
) )
name, params = next(iter(d.items())) name, params = next(iter(d.items()))
if params is None: if params is None:
@ -154,40 +202,51 @@ class GroupBase:
else: else:
if not isinstance(params, dict): if not isinstance(params, dict):
raise JenkinsJobsException( raise JenkinsJobsException(
f"{error_context}: Job/view {name} params type should be dict," f"Job/view {name!r} params type should be dict,"
f" but is {type(params)} ({params})." f" but is {params!r}.",
pos=params.pos,
) )
return cls.Spec(name, params) return cls.Spec(name, params, pos)
def _generate_items(self, root_dicts, spec_list, defaults_name, params): def _generate_items(self, root_dicts, spec_list, defaults_name, params):
for spec in spec_list: try:
item = self._pick_item(root_dicts, spec.name) for spec in spec_list:
item_params = { item = self._pick_spec_item(root_dicts, spec)
**params, item_params = LocDict.merge(
**self.params, params,
**self._my_params, self.params,
**spec.params, self._my_params,
} spec.params,
yield from item.generate_items(defaults_name, item_params) )
for job_data in item.generate_items(defaults_name, item_params):
yield (
job_data.with_context("Defined here", spec.pos).with_context(
f"In {self}", self.pos
)
)
except JenkinsJobsException as x:
raise x.with_context(f"In {self}", self.pos)
@property @property
def _my_params(self): def _my_params(self):
return {} return {}
def _pick_item(self, root_dict_list, name): def _pick_spec_item(self, root_dict_list, spec):
for roots_dict in root_dict_list: for roots_dict in root_dict_list:
try: try:
return roots_dict[name] return roots_dict[spec.name]
except KeyError: except KeyError:
pass pass
raise JenkinsJobsException( raise JenkinsJobsException(
f"{self}: Failed to find suitable job/view/template named '{name}'" f"Failed to find suitable job/view/template named '{spec.name}'",
pos=spec.pos,
) )
@dataclass @dataclass
class Group(GroupBase): class Group(GroupBase):
name: str name: str
pos: Pos
specs: list # list[Spec] specs: list # list[Spec]
params: dict params: dict

41
jenkins_jobs/roots.py
View File

@ -13,7 +13,7 @@
import logging import logging
from collections import defaultdict from collections import defaultdict
from .errors import JenkinsJobsException from .errors import Context, JenkinsJobsException
from .defaults import Defaults from .defaults import Defaults
from .job import Job, JobTemplate, JobGroup from .job import Job, JobTemplate, JobGroup
from .view import View, ViewTemplate, ViewGroup from .view import View, ViewTemplate, ViewGroup
@ -57,7 +57,7 @@ class Roots:
expanded_jobs += job.top_level_generate_items() expanded_jobs += job.top_level_generate_items()
for project in self.projects.values(): for project in self.projects.values():
expanded_jobs += project.generate_jobs() expanded_jobs += project.generate_jobs()
return self._remove_duplicates(expanded_jobs) return self._remove_duplicates(expanded_jobs, "job")
def generate_views(self): def generate_views(self):
expanded_views = [] expanded_views = []
@ -65,31 +65,44 @@ class Roots:
expanded_views += view.top_level_generate_items() expanded_views += view.top_level_generate_items()
for project in self.projects.values(): for project in self.projects.values():
expanded_views += project.generate_views() expanded_views += project.generate_views()
return self._remove_duplicates(expanded_views) return self._remove_duplicates(expanded_views, "view")
def assign(self, container, id, value, title): def assign(self, container, id, value, element_type):
if id in container: if id in container:
self._handle_dups(f"Duplicate {title}: {id}") self._handle_dups(element_type, id, value.pos, container[id].pos)
container[id] = value container[id] = value
def _remove_duplicates(self, job_list): def _remove_duplicates(self, job_or_view_list, element_type):
seen = set() seen = {}
unique_list = [] unique_list = []
# Last definition wins. # Last definition wins.
for job in reversed(job_list): for job_or_view in reversed(job_or_view_list):
name = job["name"] name = job_or_view.name
if name in seen: if name in seen:
origin = seen[name]
self._handle_dups( self._handle_dups(
f"Duplicate definitions for job {name!r} specified", element_type,
name,
job_or_view.data.pos,
origin.data.pos,
# Skip job context, leave only project context.
job_or_view.context[:-1],
origin.context[:-1],
) )
else: else:
unique_list.append(job) unique_list.append(job_or_view)
seen.add(name) seen[name] = job_or_view
return unique_list[::-1] return unique_list[::-1]
def _handle_dups(self, message): def _handle_dups(
self, element_type, id, pos, origin_pos, ctx=None, origin_ctx=None
):
message = f"Duplicate {element_type}: {id!r}"
if self._allow_duplicates: if self._allow_duplicates:
logger.warning(message) logger.warning(message)
else: else:
logger.error(message) logger.error(message)
raise JenkinsJobsException(message) ctx = [*(ctx or []), Context(message, pos), *(origin_ctx or [])]
raise JenkinsJobsException(
f"Previous {element_type} definition", origin_pos, ctx
)

60
jenkins_jobs/view.py
View File

@ -12,6 +12,8 @@
from dataclasses import dataclass from dataclasses import dataclass
from .errors import JenkinsJobsException
from .loc_loader import LocDict
from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group
from .defaults import split_contents_params, view_contents_keys from .defaults import split_contents_params, view_contents_keys
@ -21,14 +23,14 @@ class ViewBase(RootBase):
view_type: str view_type: str
@classmethod @classmethod
def from_dict(cls, config, roots, expander, data): def from_dict(cls, config, roots, expander, data, pos):
keep_descriptions = config.yamlparser["keep_descriptions"] keep_descriptions = config.yamlparser["keep_descriptions"]
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
id = d.pop("id", None) id = d.pop_loc_string("id", None)
description = d.pop("description", None) description = d.pop_loc_string("description", None)
defaults = d.pop("defaults", "global") defaults = d.pop_loc_string("defaults", "global")
view_type = d.pop("view-type", "list") view_type = d.pop_loc_string("view-type", "list")
contents, params = split_contents_params(d, view_contents_keys) contents, params = split_contents_params(d, view_contents_keys)
return cls( return cls(
roots.defaults, roots.defaults,
@ -36,6 +38,7 @@ class ViewBase(RootBase):
keep_descriptions, keep_descriptions,
id, id,
name, name,
pos,
description, description,
defaults, defaults,
params, params,
@ -44,26 +47,34 @@ class ViewBase(RootBase):
) )
def _as_dict(self): def _as_dict(self):
return { return LocDict.merge(
"name": self.name, {
"view-type": self.view_type, "name": self.name,
**self.contents, "view-type": self.view_type,
} },
self.contents,
)
class View(ViewBase, NonTemplateRootMixin): class View(ViewBase, NonTemplateRootMixin):
@classmethod @classmethod
def add(cls, config, roots, expander, param_expander, data): def add(cls, config, roots, expander, param_expander, data, pos):
view = cls.from_dict(config, roots, expander, data) view = cls.from_dict(config, roots, expander, data, pos)
roots.assign(roots.views, view.id, view, "view") roots.assign(roots.views, view.id, view, "view")
def __str__(self):
return f"view {self.name!r}"
class ViewTemplate(ViewBase, TemplateRootMixin): class ViewTemplate(ViewBase, TemplateRootMixin):
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
template = cls.from_dict(config, roots, params_expander, data) template = cls.from_dict(config, roots, params_expander, data, pos)
roots.assign(roots.view_templates, template.id, template, "view template") roots.assign(roots.view_templates, template.id, template, "view template")
def __str__(self):
return f"view template {self.name!r}"
@dataclass @dataclass
class ViewGroup(Group): class ViewGroup(Group):
@ -71,15 +82,16 @@ class ViewGroup(Group):
_view_templates: dict _view_templates: dict
@classmethod @classmethod
def add(cls, config, roots, expander, params_expander, data): def add(cls, config, roots, expander, params_expander, data, pos):
d = {**data} d = data.copy()
name = d.pop("name") name = d.pop_required_loc_string("name")
view_specs = [ try:
cls._spec_from_dict(item, error_context=f"View group {name}") view_specs = cls._specs_from_list(d.pop("views", None))
for item in d.pop("views") except JenkinsJobsException as x:
] raise x.with_context(f"In view {name!r}", pos=pos)
group = cls( group = cls(
name, name,
pos,
view_specs, view_specs,
d, d,
roots.views, roots.views,
@ -88,7 +100,7 @@ class ViewGroup(Group):
roots.assign(roots.view_groups, group.name, group, "view group") roots.assign(roots.view_groups, group.name, group, "view group")
def __str__(self): def __str__(self):
return f"View group {self.name}" return f"view group {self.name!r}"
@property @property
def _root_dicts(self): def _root_dicts(self):

9
jenkins_jobs/xml_config.py
View File

@ -21,7 +21,7 @@ import sys
from xml.dom import minidom from xml.dom import minidom
import xml.etree.ElementTree as XML import xml.etree.ElementTree as XML
from jenkins_jobs import errors from jenkins_jobs.errors import JenkinsJobsException
__all__ = ["XmlJobGenerator", "XmlJob"] __all__ = ["XmlJobGenerator", "XmlJob"]
@ -83,7 +83,10 @@ class XmlGenerator(object):
def generateXML(self, data_list): def generateXML(self, data_list):
xml_objs = [] xml_objs = []
for data in data_list: for data in data_list:
xml_objs.append(self._getXMLForData(data)) try:
xml_objs.append(self._getXMLForData(data.data))
except JenkinsJobsException as x:
raise x.with_ctx_list(data.context)
return xml_objs return xml_objs
def _getXMLForData(self, data): def _getXMLForData(self, data):
@ -104,7 +107,7 @@ class XmlGenerator(object):
ep.name ep.name
for ep in pkg_resources.iter_entry_points(group=self.entry_point_group) for ep in pkg_resources.iter_entry_points(group=self.entry_point_group)
] ]
raise errors.JenkinsJobsException( raise JenkinsJobsException(
"Unrecognized {}: {} (supported types are: {})".format( "Unrecognized {}: {} (supported types are: {})".format(
self.kind_attribute, kind, ", ".join(names) self.kind_attribute, kind, ", ".join(names)
) )

175
jenkins_jobs/yaml_objects.py
View File

@ -216,6 +216,7 @@ Examples:
import abc import abc
import os.path import os.path
import logging import logging
import traceback
import sys import sys
from pathlib import Path from pathlib import Path
@ -223,49 +224,50 @@ import jinja2
import jinja2.meta import jinja2.meta
import yaml import yaml
from .errors import JenkinsJobsException from .errors import Context, JenkinsJobsException
from .loc_loader import LocList
from .position import Pos
from .formatter import CustomFormatter, enum_str_format_required_params from .formatter import CustomFormatter, enum_str_format_required_params
logger = logging.getLogger(__name__)
if sys.version_info >= (3, 8): if sys.version_info >= (3, 8):
from functools import cached_property from functools import cached_property
else: else:
from functools import lru_cache from .cached_property import cached_property
# cached_property was introduced in python 3.8. logger = logging.getLogger(__name__)
# Recipe from https://stackoverflow.com/a/19979379
def cached_property(fn):
return property(lru_cache()(fn))
class BaseYamlObject(metaclass=abc.ABCMeta): class BaseYamlObject(metaclass=abc.ABCMeta):
@staticmethod @staticmethod
def path_list_from_node(loader, node): def path_list_from_node(loader, node):
if isinstance(node, yaml.ScalarNode): if isinstance(node, yaml.ScalarNode):
return [loader.construct_yaml_str(node)] return LocList(
[loader.construct_yaml_str(node)],
value_pos=[loader.pos_from_node(node)],
)
elif isinstance(node, yaml.SequenceNode): elif isinstance(node, yaml.SequenceNode):
return loader.construct_sequence(node) return LocList(
loader.construct_sequence(node),
value_pos=[loader.pos_from_node(n) for n in node.value],
)
else: else:
raise yaml.constructor.ConstructorError( raise JenkinsJobsException(
None, f"Expected either a sequence or scalar node, but found {node.id}",
None, pos=loader.pos_from_node(node),
f"expected either a sequence or scalar node, but found {node.id}",
node.start_mark,
) )
@classmethod @classmethod
def from_yaml(cls, loader, node): def from_yaml(cls, loader, node):
value = loader.construct_yaml_str(node) value = loader.construct_yaml_str(node)
return cls(loader.jjb_config, loader, value) return cls(loader.jjb_config, loader, loader.pos_from_node(node), value)
def __init__(self, jjb_config, loader): def __init__(self, jjb_config, loader, pos):
self._search_path = jjb_config.yamlparser["include_path"] self._search_path = jjb_config.yamlparser["include_path"]
if loader.source_path: if loader.source_path:
# Loaded from a file, find includes beside it too. # Loaded from a file, find includes beside it too.
self._search_path.append(os.path.dirname(loader.source_path)) self._search_path.append(os.path.dirname(loader.source_path))
self._loader = loader self._loader = loader
self._pos = pos
allow_empty = jjb_config.yamlparser["allow_empty_variables"] allow_empty = jjb_config.yamlparser["allow_empty_variables"]
self._formatter = CustomFormatter(allow_empty) self._formatter = CustomFormatter(allow_empty)
@ -278,7 +280,7 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
"""Expand object and substitute template parameters""" """Expand object and substitute template parameters"""
return self.expand(expander, params) return self.expand(expander, params)
def _find_file(self, rel_path): def _find_file(self, rel_path, pos):
search_path = self._search_path search_path = self._search_path
if "." not in search_path: if "." not in search_path:
search_path.append(".") search_path.append(".")
@ -288,37 +290,60 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
if candidate.is_file(): if candidate.is_file():
logger.debug("Including file %r from path %r", str(rel_path), str(dir)) logger.debug("Including file %r from path %r", str(rel_path), str(dir))
return candidate return candidate
dir_list_str = ",".join(str(d) for d in dir_list)
raise JenkinsJobsException( raise JenkinsJobsException(
f"File {rel_path} does not exist on any of include directories:" f"File {rel_path} does not exist in any of include directories: {dir_list_str}",
f" {','.join([str(d) for d in dir_list])}" pos=pos,
) )
def _expand_path_list(self, path_list, *args):
for idx, path in enumerate(path_list):
yield self._expand_path(path, path_list.value_pos[idx], *args)
def _subst_path_list(self, path_list, *args):
for idx, path in enumerate(path_list):
yield self._subst_path(path, path_list.value_pos[idx], *args)
class J2BaseYamlObject(BaseYamlObject): class J2BaseYamlObject(BaseYamlObject):
def __init__(self, jjb_config, loader): def __init__(self, jjb_config, loader, pos):
super().__init__(jjb_config, loader) super().__init__(jjb_config, loader, pos)
self._jinja2_env = jinja2.Environment( self._jinja2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(self._search_path), loader=jinja2.FileSystemLoader(self._search_path),
undefined=jinja2.StrictUndefined, undefined=jinja2.StrictUndefined,
) )
@staticmethod def _render_template(self, pos, template_text, template, params):
def _render_template(template_text, template, params):
try: try:
return template.render(params) return template.render(params)
except jinja2.UndefinedError as x: except jinja2.UndefinedError as x:
# Jinja2 adds fake traceback entry with template line number.
tb = traceback.extract_tb(x.__traceback__)
line_ofs = tb[-1].lineno - 1 # traceback lineno starts with 1.
lines = template_text.splitlines()
start_ofs = pos.body.index(lines[0])
# Examples for pre_pad: '!j2: \n<indent spaces>', '!j2: '.
pre_pad = pos.body[:start_ofs]
# Shift position to reflect template position inside yaml file:
if "\n" in pre_pad:
pos = pos.with_offset(line_ofs=1)
column_ofs = 0
else:
column_ofs = start_ofs
# Move position to error inside template:
pos = pos.with_offset(line_ofs, column_ofs)
pos = pos.with_contents_start()
if len(template_text) > 40: if len(template_text) > 40:
text = template_text[:40] + "..." text = template_text[:40] + "..."
else: else:
text = template_text text = template_text
raise JenkinsJobsException( context = Context(f"While formatting jinja2 template {text!r}", self._pos)
f"While formatting jinja2 template {text!r}: {x}" raise JenkinsJobsException(str(x), pos=pos, ctx=[context])
)
class J2Template(J2BaseYamlObject): class J2Template(J2BaseYamlObject):
def __init__(self, jjb_config, loader, template_text): def __init__(self, jjb_config, loader, pos, template_text):
super().__init__(jjb_config, loader) super().__init__(jjb_config, loader, pos)
self._template_text = template_text self._template_text = template_text
self._template = self._jinja2_env.from_string(template_text) self._template = self._jinja2_env.from_string(template_text)
@ -328,7 +353,9 @@ class J2Template(J2BaseYamlObject):
return jinja2.meta.find_undeclared_variables(ast) return jinja2.meta.find_undeclared_variables(ast)
def _render(self, params): def _render(self, params):
return self._render_template(self._template_text, self._template, params) return self._render_template(
self._pos, self._template_text, self._template, params
)
class J2String(J2Template): class J2String(J2Template):
@ -343,8 +370,11 @@ class J2Yaml(J2Template):
def expand(self, expander, params): def expand(self, expander, params):
text = self._render(params) text = self._render(params)
data = self._loader.load(text) data = self._loader.load(text, source_path="<expanded j2-yaml>")
return expander.expand(data, params) try:
return expander.expand(data, params)
except JenkinsJobsException as x:
raise x.with_context("In expanded !j2-yaml:", self._pos)
class IncludeJinja2(J2BaseYamlObject): class IncludeJinja2(J2BaseYamlObject):
@ -353,10 +383,10 @@ class IncludeJinja2(J2BaseYamlObject):
@classmethod @classmethod
def from_yaml(cls, loader, node): def from_yaml(cls, loader, node):
path_list = cls.path_list_from_node(loader, node) path_list = cls.path_list_from_node(loader, node)
return cls(loader.jjb_config, loader, path_list) return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list)
def __init__(self, jjb_config, loader, path_list): def __init__(self, jjb_config, loader, pos, path_list):
super().__init__(jjb_config, loader) super().__init__(jjb_config, loader, pos)
self._path_list = path_list self._path_list = path_list
@property @property
@ -364,91 +394,98 @@ class IncludeJinja2(J2BaseYamlObject):
return [] return []
def expand(self, expander, params): def expand(self, expander, params):
return "\n".join( return "\n".join(self._expand_path_list(self._path_list, expander, params))
self._expand_path(expander, params, path) for path in self._path_list
)
def _expand_path(self, expander, params, path_template): def _expand_path(self, path_template, pos, expander, params):
rel_path = self._formatter.format(path_template, **params) rel_path = self._formatter.format(path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
template_text = full_path.read_text() template_text = full_path.read_text()
template = self._jinja2_env.from_string(template_text) template = self._jinja2_env.from_string(template_text)
return self._render_template(template_text, template, params) pos = Pos.from_file(full_path, template_text)
try:
return self._render_template(pos, template_text, template, params)
except JenkinsJobsException as x:
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
class IncludeBaseObject(BaseYamlObject): class IncludeBaseObject(BaseYamlObject):
@classmethod @classmethod
def from_yaml(cls, loader, node): def from_yaml(cls, loader, node):
path_list = cls.path_list_from_node(loader, node) path_list = cls.path_list_from_node(loader, node)
return cls(loader.jjb_config, loader, path_list) return cls(loader.jjb_config, loader, loader.pos_from_node(node), path_list)
def __init__(self, jjb_config, loader, path_list): def __init__(self, jjb_config, loader, pos, path_list):
super().__init__(jjb_config, loader) super().__init__(jjb_config, loader, pos)
self._path_list = path_list self._path_list = path_list
@property @property
def required_params(self): def required_params(self):
for path in self._path_list: for idx, path in enumerate(self._path_list):
yield from enum_str_format_required_params(path) yield from enum_str_format_required_params(
path, pos=self._path_list.value_pos[idx]
)
class YamlInclude(IncludeBaseObject): class YamlInclude(IncludeBaseObject):
yaml_tag = "!include:" yaml_tag = "!include:"
def expand(self, expander, params): def expand(self, expander, params):
yaml_list = [ yaml_list = list(self._expand_path_list(self._path_list, expander, params))
self._expand_path(expander, params, path) for path in self._path_list
]
if len(yaml_list) == 1: if len(yaml_list) == 1:
return yaml_list[0] return yaml_list[0]
else: else:
return "\n".join(yaml_list) return "\n".join(yaml_list)
def _expand_path(self, expander, params, path_template): def _expand_path(self, path_template, pos, expander, params):
rel_path = self._formatter.format(path_template, **params) rel_path = self._formatter.format(path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
text = full_path.read_text() data = self._loader.load_path(full_path)
data = self._loader.load(text) try:
return expander.expand(data, params) return expander.expand(data, params)
except JenkinsJobsException as x:
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
class IncludeRawBase(IncludeBaseObject): class IncludeRawBase(IncludeBaseObject):
def expand(self, expander, params): def expand(self, expander, params):
return "\n".join(self._expand_path(path, params) for path in self._path_list) return "\n".join(self._expand_path_list(self._path_list, params))
def subst(self, expander, params): def subst(self, expander, params):
return "\n".join(self._subst_path(path, params) for path in self._path_list) return "\n".join(self._subst_path_list(self._path_list, params))
class IncludeRaw(IncludeRawBase): class IncludeRaw(IncludeRawBase):
yaml_tag = "!include-raw:" yaml_tag = "!include-raw:"
def _expand_path(self, rel_path_template, params): def _expand_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params) rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
return full_path.read_text() return full_path.read_text()
def _subst_path(self, rel_path_template, params): def _subst_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params) rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
template = full_path.read_text() template = full_path.read_text()
return self._formatter.format(template, **params) try:
return self._formatter.format(template, **params)
except JenkinsJobsException as x:
raise x.with_context(f"In included file {str(full_path)!r}", pos=self._pos)
class IncludeRawEscape(IncludeRawBase): class IncludeRawEscape(IncludeRawBase):
yaml_tag = "!include-raw-escape:" yaml_tag = "!include-raw-escape:"
def _expand_path(self, rel_path_template, params): def _expand_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params) rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
text = full_path.read_text() text = full_path.read_text()
# Backward compatibility: # Backward compatibility:
# if used inside job or macro without parameters, curly braces are duplicated. # if used inside job or macro without parameters, curly braces are duplicated.
return text.replace("{", "{{").replace("}", "}}") return text.replace("{", "{{").replace("}", "}}")
def _subst_path(self, rel_path_template, params): def _subst_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params) rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path) full_path = self._find_file(rel_path, pos)
return full_path.read_text() return full_path.read_text()
@ -459,12 +496,10 @@ class YamlListJoin:
def from_yaml(cls, loader, node): def from_yaml(cls, loader, node):
value = loader.construct_sequence(node, deep=True) value = loader.construct_sequence(node, deep=True)
if len(value) != 2: if len(value) != 2:
raise yaml.constructor.ConstructorError( raise JenkinsJobsException(
None,
None,
"Join value should contain 2 elements: delimiter and string list," "Join value should contain 2 elements: delimiter and string list,"
f" but contains {len(value)} elements: {value!r}", f" but contains {len(value)} elements: {value!r}",
node.start_mark, pos=loader.pos_from_node(node),
) )
delimiter, seq = value delimiter, seq = value
return delimiter.join(seq) return delimiter.join(seq)

6
tests/duplicates/job_fixtures/exception_duplicates001.error Normal file
View File

@ -0,0 +1,6 @@
exception_duplicates001.yaml:7:3: Duplicate job: 'duplicate001'
- job:
^
exception_duplicates001.yaml:1:3: Previous job definition
- job:
^

12
tests/duplicates/job_fixtures/exception_duplicates002.error Normal file
View File

@ -0,0 +1,12 @@
exception_duplicates002.yaml:14:3: Duplicate job: 'duplicates002_1.1'
- job:
^
exception_duplicates002.yaml:1:3: In project 'duplicates'
- project:
^
exception_duplicates002.yaml:6:11: Defined here
- 'duplicates002_{version}'
^
exception_duplicates002.yaml:8:3: Previous job definition
- job-template:
^

6
tests/duplicates/job_fixtures/exception_job_group001.error Normal file
View File

@ -0,0 +1,6 @@
exception_job_group001.yaml:14:3: Duplicate job group: 'group-1'
- job-group:
^
exception_job_group001.yaml:11:3: Previous job group definition
- job-group:
^

6
tests/duplicates/job_fixtures/exception_macros001.error Normal file
View File

@ -0,0 +1,6 @@
exception_macros001.yaml:9:3: Duplicate macro: 'project-scm'
- scm:
^
exception_macros001.yaml:1:3: Previous macro definition
- scm:
^

18
tests/duplicates/job_fixtures/exception_projects001.error Normal file
View File

@ -0,0 +1,18 @@
exception_projects001.yaml:1:3: In project 'duplicates'
- project:
^
exception_projects001.yaml:6:11: Defined here
- 'duplicates002_1.1'
^
exception_projects001.yaml:9:3: Duplicate job: 'duplicates002_1.1'
- job-template:
^
exception_projects001.yaml:1:3: In project 'duplicates'
- project:
^
exception_projects001.yaml:7:11: Defined here
- 'duplicates002_1.1'
^
exception_projects001.yaml:9:3: Previous job definition
- job-template:
^

30
tests/duplicates/job_fixtures/exception_projects002.error Normal file
View File

@ -0,0 +1,30 @@
exception_projects002.yaml:1:3: In project 'duplicate-groups'
- project:
^
exception_projects002.yaml:4:9: Defined here
- 'group-001'
^
exception_projects002.yaml:7:3: In job group 'group-001'
- job-group:
^
exception_projects002.yaml:10:9: Defined here
- dummy-job
^
exception_projects002.yaml:12:3: Duplicate job: 'dummy-job'
- job-template:
^
exception_projects002.yaml:1:3: In project 'duplicate-groups'
- project:
^
exception_projects002.yaml:5:9: Defined here
- 'group-001'
^
exception_projects002.yaml:7:3: In job group 'group-001'
- job-group:
^
exception_projects002.yaml:10:9: Defined here
- dummy-job
^
exception_projects002.yaml:12:3: Previous job definition
- job-template:
^

18
tests/duplicates/job_fixtures/exception_projects003.error Normal file
View File

@ -0,0 +1,18 @@
exception_projects003.yaml:1:3: In project 'duplicate-templates'
- project:
^
exception_projects003.yaml:4:11: Defined here
- '{name}-001'
^
exception_projects003.yaml:7:3: Duplicate job: 'duplicate-templates-001'
- job-template:
^
exception_projects003.yaml:1:3: In project 'duplicate-templates'
- project:
^
exception_projects003.yaml:5:11: Defined here
- '{name}-001'
^
exception_projects003.yaml:7:3: Previous job definition
- job-template:
^

6
tests/duplicates/job_fixtures/exception_templates001.error Normal file
View File

@ -0,0 +1,6 @@
exception_templates001.yaml:12:3: Duplicate job template: '{name}-001'
- job-template:
^
exception_templates001.yaml:6:3: Previous job template definition
- job-template:
^

9
tests/duplicates/test_job_duplicates.py
View File

@ -33,10 +33,13 @@ def scenario(request):
return request.param return request.param
def test_yaml_snippet(scenario, check_job): def test_yaml_snippet(scenario, expected_error, check_job):
if scenario.in_path.name.startswith("exception_"): if scenario.error_path.exists():
with pytest.raises(JenkinsJobsException) as excinfo: with pytest.raises(JenkinsJobsException) as excinfo:
check_job() check_job()
assert str(excinfo.value).startswith("Duplicate ") error = "\n".join(excinfo.value.lines)
print()
print(error)
assert error.replace(str(fixtures_dir) + "/", "") == expected_error
else: else:
check_job() check_job()

9
tests/duplicates/test_view_duplicates.py
View File

@ -18,10 +18,13 @@ def scenario(request):
return request.param return request.param
def test_yaml_snippet(scenario, check_view): def test_yaml_snippet(scenario, expected_error, check_view):
if scenario.in_path.name.startswith("exception_"): if scenario.error_path.exists():
with pytest.raises(JenkinsJobsException) as excinfo: with pytest.raises(JenkinsJobsException) as excinfo:
check_view() check_view()
assert str(excinfo.value).startswith("Duplicate ") error = "\n".join(excinfo.value.lines)
print()
print(error)
assert error.replace(str(fixtures_dir) + "/", "") == expected_error
else: else:
check_view() check_view()

6
tests/duplicates/view_fixtures/exception_duplicate_view.error Normal file
View File

@ -0,0 +1,6 @@
exception_duplicate_view.yaml:5:3: Duplicate view: 'duplicate-view'
- view:
^
exception_duplicate_view.yaml:1:3: Previous view definition
- view:
^

6
tests/duplicates/view_fixtures/exception_duplicate_view_template.error Normal file
View File

@ -0,0 +1,6 @@
exception_duplicate_view_template.yaml:5:3: Duplicate view template: 'duplicate-view-template'
- view-template:
^
exception_duplicate_view_template.yaml:1:3: Previous view template definition
- view-template:
^

18
tests/duplicates/view_fixtures/exception_duplicate_views_in_project.error Normal file
View File

@ -0,0 +1,18 @@
exception_duplicate_views_in_project.yaml:5:3: In project 'sample-project'
- project:
^
exception_duplicate_views_in_project.yaml:8:9: Defined here
- sample-template
^
exception_duplicate_views_in_project.yaml:1:3: Duplicate view: 'sample-template'
- view-template:
^
exception_duplicate_views_in_project.yaml:5:3: In project 'sample-project'
- project:
^
exception_duplicate_views_in_project.yaml:9:9: Defined here
- sample-template
^
exception_duplicate_views_in_project.yaml:1:3: Previous view definition
- view-template:
^

5
tests/formatter/test_formatter.py
View File

@ -1,6 +1,7 @@
import pytest import pytest
from jinja2 import StrictUndefined from jinja2 import StrictUndefined
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.formatter import ( from jenkins_jobs.formatter import (
CustomFormatter, CustomFormatter,
enum_str_format_required_params, enum_str_format_required_params,
@ -144,7 +145,7 @@ def test_format(format, vars, used_vars, expected_defaults, expected_result):
def test_used_params( def test_used_params(
format, vars, expected_used_vars, expected_defaults, expected_result format, vars, expected_used_vars, expected_defaults, expected_result
): ):
used_vars = set(enum_str_format_required_params(format)) used_vars = set(enum_str_format_required_params(format, pos=None))
assert used_vars == set(expected_used_vars) assert used_vars == set(expected_used_vars)
@ -193,7 +194,7 @@ positional_cases = [
@pytest.mark.parametrize("format", positional_cases) @pytest.mark.parametrize("format", positional_cases)
def test_positional_args(format): def test_positional_args(format):
formatter = CustomFormatter(allow_empty=False) formatter = CustomFormatter(allow_empty=False)
with pytest.raises(RuntimeError) as excinfo: with pytest.raises(JenkinsJobsException) as excinfo:
list(formatter.enum_required_params(format)) list(formatter.enum_required_params(format))
message = f"Positional format arguments are not supported: {format!r}" message = f"Positional format arguments are not supported: {format!r}"
assert str(excinfo.value) == message assert str(excinfo.value) == message

2
tests/formatter/test_jinja2.py
View File

@ -17,5 +17,5 @@ cases = [
def test_jinja2_required_params(format, expected_used_params): def test_jinja2_required_params(format, expected_used_params):
config = JJBConfig() config = JJBConfig()
loader = Mock(source_path=None) loader = Mock(source_path=None)
template = J2String(config, loader, format) template = J2String(config, loader, pos=None, template_text=format)
assert template.required_params == set(expected_used_params) assert template.required_params == set(expected_used_params)

29
tests/loader/loc_fixtures/sample_01.yaml Normal file
View File

@ -0,0 +1,29 @@
- job_template:
name: sample-job-1
builders: &job_builders
- shell: |
#!/usr/bin/env bash -xe
echo this is sample bash script
and this is it's third line
- sample_macro: &macro_params
param_1: value_1
param_2: value_2
- job_template:
name: sample-job-2
builders: *job_builders
- job_template:
name: sample-job-3
builders:
- sample_macro:
<<: *macro_params
param_3: value_3
- project:
name: sample-project
param_1: sample_value
jobs:
- sample-job-1
- sample-job-2
- sample-job-3

4
tests/loader/test_loader.py
View File

@ -62,7 +62,7 @@ def test_include(scenario, jjb_config, expected_output):
roots = Roots(jjb_config) roots = Roots(jjb_config)
load_files(jjb_config, roots, [scenario.in_path]) load_files(jjb_config, roots, [scenario.in_path])
job_data_list = roots.generate_jobs() job_data_list = [j.data for j in roots.generate_jobs()]
pretty_json = json.dumps(job_data_list, indent=4) pretty_json = json.dumps(job_data_list, indent=4)
print(pretty_json) print(pretty_json)
assert pretty_json == expected_output.strip() assert pretty_json == expected_output.strip()
@ -198,4 +198,4 @@ def test_retain_anchors_enabled_j2_yaml():
registry = ModuleRegistry(config, None) registry = ModuleRegistry(config, None)
registry.set_macros(roots.macros) registry.set_macros(roots.macros)
jobs = roots.generate_jobs() jobs = roots.generate_jobs()
assert "docker run ubuntu:latest" == jobs[0]["builders"][0]["shell"] assert "docker run ubuntu:latest" == jobs[0].data["builders"][0]["shell"]

72
tests/loader/test_locations.py Normal file
View File

@ -0,0 +1,72 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pathlib import Path
from jenkins_jobs.loc_loader import LocLoader
fixtures_dir = Path(__file__).parent / "loc_fixtures"
def test_location():
print()
path = fixtures_dir / "sample_01.yaml"
loader = LocLoader(path.read_text(), str(path))
data = loader.get_single_data()
b = data[0]["job_template"]["builders"][0]
print(type(b), b.pos)
print(b)
print(b.pos.snippet)
b = data[0]["job_template"]["builders"][1]["sample_macro"]
print(type(b), b.pos)
print(b)
print("items", b.value_pos)
for key, pos in b.value_pos.items():
print("item:", key, pos)
print(pos.snippet)
b = data[0]["job_template"]["builders"]
print(type(b), b.pos)
print(b)
print("items", b.value_pos)
for idx, pos in enumerate(b.value_pos):
print("item:", idx, pos)
print("--- sample-job-2 builders -------------------------------------------")
b = data[1]["job_template"]["builders"]
print(type(b), b.pos)
print(b)
print("items", b.value_pos)
for idx, pos in enumerate(b.value_pos):
print("item:", idx, pos)
print(pos.snippet)
print("--- sample-job-2 sample_macro ---------------------------------------")
sample_macro = data[1]["job_template"]["builders"][1]["sample_macro"]
param_2_pos = sample_macro.value_pos["param_2"]
print(param_2_pos)
print(param_2_pos.snippet)
assert param_2_pos.line == 9
assert param_2_pos.column == 19
assert param_2_pos.snippet.splitlines()[0].strip() == "param_2: value_2"
print("--- third template -------------------------------------------------")
b = data[2]["job_template"]["builders"][0]["sample_macro"]
print(type(b), b.pos)
print(b)
print("items", b.value_pos)
for key, pos in b.key_pos.items():
print("keys for item:", key, pos)
for key, pos in b.value_pos.items():
print("values for item:", key, pos)

7
tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type001.error
View File

@ -1 +1,6 @@
'['test']' is an invalid value for attribute name.trigger-build-on-pr-comment scm_github_comment_plugin_invalid_type001.yaml:12:19: For attribute 'trigger-build-on-pr-comment'
- trigger-build-on-pr-comment:
^
scm_github_comment_plugin_invalid_type001.yaml:13:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-comment
- test
^

7
tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type002.error
View File

@ -1 +1,6 @@
'['test']' is an invalid value for attribute name.trigger-build-on-pr-review scm_github_comment_plugin_invalid_type002.yaml:14:19: For attribute 'trigger-build-on-pr-review'
- trigger-build-on-pr-review:
^
scm_github_comment_plugin_invalid_type002.yaml:15:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-review
- test
^

7
tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type003.error
View File

@ -1 +1,6 @@
'['test']' is an invalid value for attribute name.trigger-build-on-pr-update scm_github_comment_plugin_invalid_type003.yaml:16:19: For attribute 'trigger-build-on-pr-update'
- trigger-build-on-pr-update:
^
scm_github_comment_plugin_invalid_type003.yaml:17:21: '['test']' is an invalid value for attribute name.trigger-build-on-pr-update
- test
^

7
tests/multibranch/error_fixtures/scm_github_comment_plugin_invalid_type004.error
View File

@ -1 +1,6 @@
'true' is an invalid value for attribute name.trigger-build-on-pr-update scm_github_comment_plugin_invalid_type004.yaml:16:19: For attribute 'trigger-build-on-pr-update'
- trigger-build-on-pr-update: "true"
^
scm_github_comment_plugin_invalid_type004.yaml:16:47: 'true' is an invalid value for attribute name.trigger-build-on-pr-update
- trigger-build-on-pr-update: "true"
^

4
tests/multibranch/error_fixtures/scm_github_comment_plugin_missing_comment.error
View File

@ -1 +1,3 @@
Missing trigger-build-on-pr-comment[comment] from an instance of 'name' scm_github_comment_plugin_missing_comment.yaml:12:19: Missing trigger-build-on-pr-comment[comment] from an instance of 'name'
- trigger-build-on-pr-comment:
^

8
tests/multibranch/test_errors.py
View File

@ -17,6 +17,7 @@ from pathlib import Path
import pytest import pytest
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.modules import project_multibranch from jenkins_jobs.modules import project_multibranch
from tests.enum_scenarios import scenario_list from tests.enum_scenarios import scenario_list
@ -32,6 +33,9 @@ def scenario(request):
def test_error(check_generator, expected_error): def test_error(check_generator, expected_error):
with pytest.raises(Exception) as excinfo: with pytest.raises(JenkinsJobsException) as excinfo:
check_generator(project_multibranch.WorkflowMultiBranch) check_generator(project_multibranch.WorkflowMultiBranch)
assert str(excinfo.value) == expected_error error = "\n".join(excinfo.value.lines)
print()
print(error)
assert error.replace(str(fixtures_dir) + "/", "") == expected_error

16
tests/xml_config/test_xml_config.py
View File

@ -77,10 +77,7 @@ def test_template_params(parser, registry):
with pytest.raises(Exception) as excinfo: with pytest.raises(Exception) as excinfo:
generator.generateXML(jobs) generator.generateXML(jobs)
message = ( message = "While formatting string '{branches}': Missing parameter: 'branches'"
"While expanding macro 'default-git-scm':"
" While formatting string '{branches}': Missing parameter: 'branches'"
)
assert str(excinfo.value) == message assert str(excinfo.value) == message
@ -91,10 +88,7 @@ def test_missing_j2_param(parser, registry):
with pytest.raises(Exception) as excinfo: with pytest.raises(Exception) as excinfo:
generator.generateXML(jobs) generator.generateXML(jobs)
message = ( message = "'branches' is undefined"
"While expanding macro 'default-git-scm':"
" While formatting jinja2 template '{{ branches }}': 'branches' is undefined"
)
assert str(excinfo.value) == message assert str(excinfo.value) == message
@ -105,9 +99,5 @@ def test_missing_include_j2_param(parser, registry):
with pytest.raises(Exception) as excinfo: with pytest.raises(Exception) as excinfo:
generator.generateXML(jobs) generator.generateXML(jobs)
message = ( message = "'branches' is undefined"
"While expanding macro 'a-builder':"
" While formatting jinja2 template 'echo \"Parameter branch={{ branches }} is...':"
" 'branches' is undefined"
)
assert str(excinfo.value) == message assert str(excinfo.value) == message

7
tests/yamlparser/error_fixtures/failure_formatting_indent.error
View File

@ -1 +1,6 @@
Project missing_params_for_params: Job/view dict should be single-item, but have keys ['template-requiring-param-{os}', 'os']. Missing indent? failure_formatting_indent.yaml:5:3: In project 'missing_params_for_params'
- project:
^
failure_formatting_indent.yaml:13:9: Job/view dict should be single-item, but have keys ['template-requiring-param-{os}', 'os']. Missing indent?
- 'template-requiring-param-{os}':
^

13
tests/yamlparser/error_fixtures/failure_formatting_params.error
View File

@ -1 +1,12 @@
While expanding 'flavor', used by , used by template 'template-requiring-param-{os}': While formatting string 'xenial-{bdate}': 'bdate' is undefined failure_formatting_params.yaml:5:3: In project 'missing_params_for_params'
- project:
^
failure_formatting_params.yaml:16:3: In job template 'template-requiring-param-{os}'
- job-template:
^
failure_formatting_params.yaml:9:5: While expanding parameter 'flavor'
flavor:
^
failure_formatting_params.yaml:11:9: While formatting string 'xenial-{bdate}': 'bdate' is undefined
- xenial-{bdate}
^

10
tests/yamlparser/error_fixtures/failure_formatting_template.error
View File

@ -1 +1,9 @@
While formatting string 'template-requiring-param-{os}': Missing parameter: 'os' failure_formatting_template.yaml:1:3: In project 'missing_params_for_template'
- project:
^
failure_formatting_template.yaml:6:3: In job template 'template-requiring-param-{os}'
- job-template:
^
failure_formatting_template.yaml:7:12: While formatting string 'template-requiring-param-{os}': Missing parameter: 'os'
name: 'template-requiring-param-{os}'
^

9
tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.error Normal file
View File

@ -0,0 +1,9 @@
format_positional_argument_in_job_name.yaml:6:3: In project 'sample-project'
- project:
^
format_positional_argument_in_job_name.yaml:1:3: In job template 'sample-job-{0}'
- job-template:
^
format_positional_argument_in_job_name.yaml:2:11: Positional format arguments are not supported: 'sample-job-{0}'
name: sample-job-{0}
^

9
tests/yamlparser/error_fixtures/format_positional_argument_in_job_name.yaml Normal file
View File

@ -0,0 +1,9 @@
- job-template:
name: sample-job-{0}
builders:
- shell: echo ok
- project:
name: sample-project
jobs:
- sample-job-{0}

9
tests/yamlparser/error_fixtures/format_positional_argument_in_param.error Normal file
View File

@ -0,0 +1,9 @@
format_positional_argument_in_param.yaml:6:3: In project 'sample-project'
- project:
^
format_positional_argument_in_param.yaml:1:3: In job template 'sample-job'
- job-template:
^
format_positional_argument_in_param.yaml:8:12: Positional format arguments are not supported: 'positional-format-{0}'
param: 'positional-format-{0}'
^

10
tests/yamlparser/error_fixtures/format_positional_argument_in_param.yaml Normal file
View File

@ -0,0 +1,10 @@
- job-template:
name: sample-job
builders:
- shell: echo {param}
- project:
name: sample-project
param: 'positional-format-{0}'
jobs:
- sample-job

9
tests/yamlparser/error_fixtures/include_jinja2_missing_path.error Normal file
View File

@ -0,0 +1,9 @@
include_jinja2_missing_path.yaml:6:3: In project 'sample-project'
- project:
^
include_jinja2_missing_path.yaml:1:3: In job template 'sample-job'
- job-template:
^
include_jinja2_missing_path.yaml:4:16: File missing-path.inc does not exist in any of include directories: .,fixtures-dir
- shell: !include-jinja2: missing-path.inc
^

11
tests/yamlparser/error_fixtures/missing_param_jinja2_template_direct.yaml → tests/yamlparser/error_fixtures/include_jinja2_missing_path.yaml
View File

@ -1,10 +1,9 @@
- job-template:
name: sample-job
builders:
- shell: !include-jinja2: missing-path.inc
- project: - project:
name: sample-project name: sample-project
jobs: jobs:
- sample-job - sample-job
- job-template:
name: sample-job
builders:
- shell: !j2: |
echo {{ missing_param }}

9
tests/yamlparser/error_fixtures/include_missing_path.error Normal file
View File

@ -0,0 +1,9 @@
include_missing_path.yaml:7:3: In project 'sample-project'
- project:
^
include_missing_path.yaml:1:3: In job template 'sample-job'
- job-template:
^
include_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir
!include: missing-file.sh
^

10
tests/yamlparser/error_fixtures/include_missing_path.yaml Normal file
View File

@ -0,0 +1,10 @@
- job-template:
name: sample-job
builders:
- shell:
!include: missing-file.sh
- project:
name: sample-project
jobs:
- sample-job

12
tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.error Normal file
View File

@ -0,0 +1,12 @@
include_missing_path_in_j2_yaml.yaml:12:3: In project 'sample-project'
- project:
^
include_missing_path_in_j2_yaml.yaml:3:3: In job template 'sample-job'
- job-template:
^
include_missing_path_in_j2_yaml.yaml:5:15: In expanded !j2-yaml:
builders: !j2-yaml: |
^
<expanded j2-yaml>:2:5: File missing-file.sh does not exist in any of include directories: .,fixtures-dir,.
!include: missing-file.sh
^

15
tests/yamlparser/error_fixtures/include_missing_path_in_j2_yaml.yaml Normal file
View File

@ -0,0 +1,15 @@
# Check for error handling inside expanded template.
- job-template:
name: sample-job
builders: !j2-yaml: |
{# Comment lines -#}
{# added to change templated position -#}
{# of include error -#}
- shell:
!include: missing-file.sh
- project:
name: sample-project
jobs:
- sample-job

9
tests/yamlparser/error_fixtures/include_raw_escape_missing_path.error Normal file
View File

@ -0,0 +1,9 @@
include_raw_escape_missing_path.yaml:7:3: In project 'sample-project'
- project:
^
include_raw_escape_missing_path.yaml:1:3: In job template 'sample-job'
- job-template:
^
include_raw_escape_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir
!include-raw-escape: missing-file.sh
^

10
tests/yamlparser/error_fixtures/include_raw_escape_missing_path.yaml Normal file
View File

@ -0,0 +1,10 @@
- job-template:
name: sample-job
builders:
- shell:
!include-raw-escape: missing-file.sh
- project:
name: sample-project
jobs:
- sample-job

9
tests/yamlparser/error_fixtures/include_raw_missing_path.error Normal file
View File

@ -0,0 +1,9 @@
include_raw_missing_path.yaml:7:3: In project 'sample-project'
- project:
^
include_raw_missing_path.yaml:1:3: In job template 'sample-job'
- job-template:
^
include_raw_missing_path.yaml:5:11: File missing-file.sh does not exist in any of include directories: .,fixtures-dir
!include-raw: missing-file.sh
^

10
tests/yamlparser/error_fixtures/include_raw_missing_path.yaml Normal file
View File

@ -0,0 +1,10 @@
- job-template:
name: sample-job
builders:
- shell:
!include-raw: missing-file.sh
- project:
name: sample-project
jobs:
- sample-job

12
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.error Normal file
View File

@ -0,0 +1,12 @@
incorrect_dimension_exclude_empty.yaml:6:3: In project 'sample-project'
- project:
^
incorrect_dimension_exclude_empty.yaml:1:3: In job template 'sample-job-{dimension}'
- job-template:
^
incorrect_dimension_exclude_empty.yaml:11:5: In template exclude list
exclude:
^
incorrect_dimension_exclude_empty.yaml:12:7: Expected a dict, but is empty: {}
- {}
^

14
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_empty.yaml Normal file
View File

@ -0,0 +1,14 @@
- job-template:
name: 'sample-job-{dimension}'
builders:
- shell: echo {dimension}
- project:
name: sample-project
dimension:
- first
- second
exclude:
- {}
jobs:
- 'sample-job-{dimension}'

12
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.error Normal file
View File

@ -0,0 +1,12 @@
incorrect_dimension_exclude_not_dict.yaml:6:3: In project 'sample-project'
- project:
^
incorrect_dimension_exclude_not_dict.yaml:1:3: In job template 'sample-job-{dimension}'
- job-template:
^
incorrect_dimension_exclude_not_dict.yaml:11:5: In template exclude list
exclude:
^
incorrect_dimension_exclude_not_dict.yaml:12:7: Expected a dict, but got: 'wrong-value'
- wrong-value
^

14
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_not_dict.yaml Normal file
View File

@ -0,0 +1,14 @@
- job-template:
name: 'sample-job-{dimension}'
builders:
- shell: echo {dimension}
- project:
name: sample-project
dimension:
- first
- second
exclude:
- wrong-value
jobs:
- 'sample-job-{dimension}'

12
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.error Normal file
View File

@ -0,0 +1,12 @@
incorrect_dimension_exclude_unknown_axis.yaml:6:3: In project 'sample-project'
- project:
^
incorrect_dimension_exclude_unknown_axis.yaml:1:3: In job template 'sample-job-{dimension}'
- job-template:
^
incorrect_dimension_exclude_unknown_axis.yaml:11:5: In template exclude list
exclude:
^
incorrect_dimension_exclude_unknown_axis.yaml:13:7: Unknown axis 'wrong_axis'
- wrong_axis: some-value
^

15
tests/yamlparser/error_fixtures/incorrect_dimension_exclude_unknown_axis.yaml Normal file
View File

@ -0,0 +1,15 @@
- job-template:
name: 'sample-job-{dimension}'
builders:
- shell: echo {dimension}
- project:
name: sample-project
dimension:
- first
- second
exclude:
- dimension: second
- wrong_axis: some-value
jobs:
- 'sample-job-{dimension}'

6
tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.error Normal file
View File

@ -0,0 +1,6 @@
incorrect_job_spec_multi_keys.yaml:4:3: In project 'sample-project'
- project:
^
incorrect_job_spec_multi_keys.yaml:7:9: Job/view dict should be single-item, but have keys ['sample-job', 'incorrectly_indented_parameter']. Missing indent?
- sample-job:
^

8
tests/yamlparser/error_fixtures/incorrect_job_spec_multi_keys.yaml Normal file
View File

@ -0,0 +1,8 @@
- job-template:
name: sample-job
- project:
name: sample-project
jobs:
- sample-job:
incorrectly_indented_parameter:

6
tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.error Normal file
View File

@ -0,0 +1,6 @@
incorrect_job_spec_params_not_dict.yaml:4:3: In project 'sample-project'
- project:
^
incorrect_job_spec_params_not_dict.yaml:8:11: Job/view 'sample-job' params type should be dict, but is ['abc', 'def'].
- abc
^

9
tests/yamlparser/error_fixtures/incorrect_job_spec_params_not_dict.yaml Normal file
View File

@ -0,0 +1,9 @@
- job-template:
name: sample-job
- project:
name: sample-project
jobs:
- sample-job:
- abc
- def

6
tests/yamlparser/error_fixtures/incorrect_job_spec_type.error Normal file
View File

@ -0,0 +1,6 @@
incorrect_job_spec_type.yaml:4:3: In project 'sample-project'
- project:
^
incorrect_job_spec_type.yaml:7:9: Job/view spec should name or dict, but is <class 'int'> (123). Missing indent?
- 123
^

7
tests/yamlparser/error_fixtures/incorrect_job_spec_type.yaml Normal file
View File

@ -0,0 +1,7 @@
- job-template:
name: sample-job
- project:
name: sample-project
jobs:
- 123

13
tests/yamlparser/error_fixtures/incorrect_template_dimensions.error
View File

@ -1 +1,12 @@
Invalid parameter 'stream' definition for template 'template-incorrect-args-{stream}-{os}': Expected a value or a dict with single element, but got: {'current': None, 'branch': 'current'} incorrect_template_dimensions.yaml:1:3: In project 'template_incorrect_args'
- project:
^
incorrect_template_dimensions.yaml:14:3: In job template 'template-incorrect-args-{stream}-{os}'
- job-template:
^
incorrect_template_dimensions.yaml:6:5: In pamareter 'stream' definition
stream:
^
incorrect_template_dimensions.yaml:7:9: Expected a value or a dict with single element, but got: {'current': None, 'branch': 'current'}
- current:
^

3
tests/yamlparser/error_fixtures/invalid_include_path_type.error Normal file
View File

@ -0,0 +1,3 @@
invalid_include_path_type.yaml:5:11: Expected either a sequence or scalar node, but found mapping
!include:
^

11
tests/yamlparser/error_fixtures/invalid_include_path_type.yaml Normal file
View File

@ -0,0 +1,11 @@
- job-template:
name: sample-job
builders:
- shell:
!include:
key: value
- project:
name: sample-project
jobs:
- sample-job

10
tests/yamlparser/error_fixtures/job_group_includes_missing_job.error
View File

@ -1 +1,9 @@
Job group group-1: Failed to find suitable job/view/template named 'job-2' job_group_includes_missing_job.yaml:14:3: In project 'sample-project'
- project:
^
job_group_includes_missing_job.yaml:8:3: In job group 'group-1'
- job-group:
^
job_group_includes_missing_job.yaml:12:9: Failed to find suitable job/view/template named 'job-2'
- job-2
^

6
tests/yamlparser/error_fixtures/missing_defaults_at_job.error Normal file
View File

@ -0,0 +1,6 @@
missing_defaults_at_job.yaml:1:3: In job 'sample-job'
- job:
^
missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults' but it was never defined
defaults: missing-defaults
^

8
tests/yamlparser/error_fixtures/missing_defaults_at_job.yaml Normal file
View File

@ -0,0 +1,8 @@
- job:
name: sample-job
defaults: missing-defaults
- project:
name: sample-project
jobs:
- sample-job

9
tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error Normal file
View File

@ -0,0 +1,9 @@
missing_defaults_at_job_template.yaml:5:3: In project 'sample-project'
- project:
^
missing_defaults_at_job_template.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults' but it was never defined
defaults: missing-defaults
^

8
tests/yamlparser/error_fixtures/missing_defaults_at_job_template.yaml Normal file
View File

@ -0,0 +1,8 @@
- job-template:
name: sample-job
defaults: missing-defaults
- project:
name: sample-project
jobs:
- sample-job

9
tests/yamlparser/error_fixtures/missing_defaults_at_project.error Normal file
View File

@ -0,0 +1,9 @@
missing_defaults_at_project.yaml:4:3: In project 'sample-project'
- project:
^
missing_defaults_at_project.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'global' but it was never defined
defaults: missing-defaults
^

8
tests/yamlparser/error_fixtures/missing_defaults_at_project.yaml Normal file
View File

@ -0,0 +1,8 @@
- job-template:
name: sample-job
- project:
name: sample-project
defaults: missing-defaults
jobs:
- sample-job

3
tests/yamlparser/error_fixtures/missing_job_element_name.error Normal file
View File

@ -0,0 +1,3 @@
missing_job_element_name.yaml:3:5: Missing required element: 'name'
some_thing: value
^

3
tests/yamlparser/error_fixtures/missing_job_element_name.yaml Normal file
View File

@ -0,0 +1,3 @@
# Job with no 'name' element defined.
- job:
some_thing: value

3
tests/yamlparser/error_fixtures/missing_macro_element.error Normal file
View File

@ -0,0 +1,3 @@
missing_macro_element.yaml:3:5: Missing required element: 'builders'
name: sample-builder
^

3
tests/yamlparser/error_fixtures/missing_macro_element.yaml Normal file
View File

@ -0,0 +1,3 @@
# Builder macro with no 'builders' element defined.
- builder:
name: sample-builder

15
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.error Normal file
View File

@ -0,0 +1,15 @@
missing_param_in_include_jinja2.yaml:6:3: In project 'sample-project'
- project:
^
missing_param_in_include_jinja2.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_param_in_include_jinja2.yaml:4:16: In included file 'missing_param_in_include_jinja2.inc'
- shell: !include-jinja2: missing_param_in_include_jin ...
^
missing_param_in_include_jinja2.yaml:4:16: While formatting jinja2 template '{# Sample comment #}\n#!/bin/bash\n\nif [ -...'
- shell: !include-jinja2: missing_param_in_include_jin ...
^
missing_param_in_include_jinja2.inc:5:5: 'johnny' is undefined
echo "Here is {{ johnny }}!"
^

6
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.inc Normal file
View File

@ -0,0 +1,6 @@
{# Sample comment #}
#!/bin/bash
if [ -f the-door ]; then
echo "Here is {{ johnny }}!"
fi

9
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2.yaml Normal file
View File

@ -0,0 +1,9 @@
- job-template:
name: sample-job
builders:
- shell: !include-jinja2: missing_param_in_include_jinja2.inc
- project:
name: sample-project
jobs:
- sample-job

15
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.error Normal file
View File

@ -0,0 +1,15 @@
missing_param_in_include_jinja2_format.yaml:5:3: In project 'sample-project'
- project:
^
missing_param_in_include_jinja2_format.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_param_in_include_jinja2_format.yaml:3:15: In included file 'missing_param_in_include_jinja2_format.yaml.inc'
builders: !include: missing_param_in_include_jinja2_for ...
^
missing_param_in_include_jinja2_format.yaml.inc:2:14: While formatting jinja2 template '#!/bin/bash\n\necho "hello, {{ unknown_one...'
command: !j2: |
^
missing_param_in_include_jinja2_format.yaml.inc:5:7: 'unknown_one' is undefined
echo "hello, {{ unknown_one }}!"
^

8
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml Normal file
View File

@ -0,0 +1,8 @@
- job-template:
name: sample-job
builders: !include: missing_param_in_include_jinja2_format.yaml.inc
- project:
name: sample-project
jobs:
- sample-job

5
tests/yamlparser/error_fixtures/missing_param_in_include_jinja2_format.yaml.inc Normal file
View File

@ -0,0 +1,5 @@
- shell:
command: !j2: |
#!/bin/bash
echo "hello, {{ unknown_one }}!"

10
tests/yamlparser/error_fixtures/missing_param_in_include_raw.error Normal file
View File

@ -0,0 +1,10 @@
missing_param_in_include_raw.yaml:6:3: In project 'sample-project'
- project:
^
missing_param_in_include_raw.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_param_in_include_raw.yaml:4:16: In included file 'missing_param_in_include_raw.inc'
- shell: !include-raw: missing_param_in_include_raw.inc
^
While formatting string '#!/bin/bash\necho "This one is {missing}!"\n...': Missing parameter: 'missing'

Some files were not shown because too many files have changed in this diff Show More