openstack/heat
Code Issues Proposed changes

Merge "Add yaql function"

Browse Source
This commit is contained in:
Jenkins 2016-05-24 14:04:47 +00:00 committed by Gerrit Code Review
commit 2969f0a49e
5 changed files with 211 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
doc/source/template_guide/hot_spec.rst
View File

@ -203,6 +203,27 @@ For example, Heat currently supports the following values for the
str_replace str_replace
str_split str_split
2016-10-14
----------
The key with value ``2016-10-14`` indicates that the YAML document is a HOT
template and it may contain features added and/or removed up until the
Newton release. This version also adds the yaql function which
can be used for evaluation of complex expressions. The complete list of
supported functions is::
digest
get_attr
get_file
get_param
get_resource
list_join
map_merge
repeat
resource_facade
str_replace
str_split
yaql
.. _hot_spec_parameter_groups: .. _hot_spec_parameter_groups:
Parameter groups section Parameter groups section
@ -1264,3 +1285,33 @@ For example
This resolves to a map containing ``{'k1': 'v2', 'k2': 'v2'}``. This resolves to a map containing ``{'k1': 'v2', 'k2': 'v2'}``.
Maps containing no items resolve to {}. Maps containing no items resolve to {}.
yaql
----
The ``yaql`` evaluates yaql expression on a given data.
The syntax of the ``yaql`` function is
.. code-block:: yaml
yaql:
expression: <expression>
data: <data>
For example
.. code-block:: yaml
parameters:
list_param:
type: comma_delimited_list
default: [1, 2, 3]
outputs:
max_elem:
yaql:
expression: $.data.list_param.select(int($)).max()
data:
list_param: {get_param: list_param}
max_elem output will be evaluated to 3

99
heat/engine/hot/functions.py
View File

@ -14,9 +14,12 @@
import collections import collections
import hashlib import hashlib
import itertools import itertools
import six
from oslo_config import cfg
from oslo_serialization import jsonutils from oslo_serialization import jsonutils
import six
import yaql
from yaql.language import exceptions
from heat.common import exception from heat.common import exception
from heat.common.i18n import _ from heat.common.i18n import _
@ -24,6 +27,18 @@ from heat.engine import attributes
from heat.engine.cfn import functions as cfn_funcs from heat.engine.cfn import functions as cfn_funcs
from heat.engine import function from heat.engine import function
opts = [
cfg.IntOpt('limit_iterators',
default=200,
help=_('The maximum number of elements in collection '
'expression can take for its evaluation.')),
cfg.IntOpt('memory_quota',
default=10000,
help=_('The maximum size of memory in bytes that '
'expression can take for its evaluation.'))
]
cfg.CONF.register_opts(opts, group='yaql')
class GetParam(function.Function): class GetParam(function.Function):
"""A function for resolving parameter references. """A function for resolving parameter references.
@ -697,3 +712,85 @@ class StrSplit(function.Function):
else: else:
res = split_list res = split_list
return res return res
class Yaql(function.Function):
"""A function for executing a yaql expression.
Takes the form::
yaql:
expression:
<body>
data:
<var>: <list>
Evaluates expression <body> on the given data.
"""
_parser = None
@classmethod
def get_yaql_parser(cls):
if cls._parser is None:
global_options = {
'yaql.limitIterators': cfg.CONF.yaql.limit_iterators,
'yaql.memoryQuota': cfg.CONF.yaql.memory_quota
}
cls._parser = yaql.YaqlFactory().create(global_options)
return cls._parser
def __init__(self, stack, fn_name, args):
super(Yaql, self).__init__(stack, fn_name, args)
self.parser = self.get_yaql_parser()
self.context = yaql.create_context()
if not isinstance(self.args, collections.Mapping):
raise TypeError(_('Arguments to "%s" must be a map.') %
self.fn_name)
try:
self._expression = self.args['expression']
self._data = self.args.get('data', {})
for arg in six.iterkeys(self.args):
if arg not in ['expression', 'data']:
raise KeyError
except (KeyError, TypeError):
example = ('''%s:
expression: $.data.var1.sum()
data:
var1: [3, 2, 1]''') % self.fn_name
raise KeyError(_('"%(name)s" syntax should be %(example)s') % {
'name': self.fn_name, 'example': example})
def validate_expression(self, expression):
try:
self.parser(expression)
except exceptions.YaqlException as yex:
raise ValueError(_('Bad expression %s.') % yex)
def validate(self):
super(Yaql, self).validate()
if not isinstance(self._data,
(collections.Mapping, function.Function)):
raise TypeError(_('The "data" argument to "%s" must contain '
'a map.') % self.fn_name)
if not isinstance(self._expression,
(six.string_types, function.Function)):
raise TypeError(_('The "expression" argument to %s must '
'contain a string or a '
'function.') % self.fn_name)
if isinstance(self._expression, six.string_types):
self.validate_expression(self._expression)
def result(self):
data = function.resolve(self._data)
if not isinstance(data, collections.Mapping):
raise TypeError(_('The "data" argument to "%s" must contain '
'a map.') % self.fn_name)
ctxt = {'data': data}
self.context['$'] = ctxt
if not isinstance(self._expression, six.string_types):
self._expression = function.resolve(self._expression)
self.validate_expression(self._expression)
return self.parser(self._expression).evaluate(context=self.context)

3
heat/engine/hot/template.py
View File

@ -411,6 +411,9 @@ class HOTemplate20161014(HOTemplate20160408):
'resource_facade': hot_funcs.ResourceFacade, 'resource_facade': hot_funcs.ResourceFacade,
'str_replace': hot_funcs.ReplaceJson, 'str_replace': hot_funcs.ReplaceJson,
# functions added since 20161014
'yaql': hot_funcs.Yaql,
# functions added since 20151015 # functions added since 20151015
'map_merge': hot_funcs.MapMerge, 'map_merge': hot_funcs.MapMerge,

58
heat/tests/test_hot.py
View File

@ -57,6 +57,10 @@ hot_mitaka_tpl_empty = template_format.parse('''
heat_template_version: 2016-04-08 heat_template_version: 2016-04-08
''') ''')
hot_newton_tpl_empty = template_format.parse('''
heat_template_version: 2016-10-14
''')
hot_tpl_empty_sections = template_format.parse(''' hot_tpl_empty_sections = template_format.parse('''
heat_template_version: 2013-05-23 heat_template_version: 2013-05-23
parameters: parameters:
@ -833,6 +837,60 @@ class HOTemplateTest(common.HeatTestCase):
self.assertEqual('role1', resolved['role1']) self.assertEqual('role1', resolved['role1'])
self.assertEqual('role2', resolved['role2']) self.assertEqual('role2', resolved['role2'])
def test_yaql(self):
snippet = {'yaql': {'expression': '$.data.var1.sum()',
'data': {'var1': [1, 2, 3, 4]}}}
tmpl = template.Template(hot_newton_tpl_empty)
stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl)
resolved = self.resolve(snippet, tmpl, stack=stack)
self.assertEqual(10, resolved)
def test_yaql_invalid_data(self):
snippet = {'yaql': {'expression': '$.data.var1.sum()',
'data': 'mustbeamap'}}
tmpl = template.Template(hot_newton_tpl_empty)
msg = 'The "data" argument to "yaql" must contain a map.'
self.assertRaisesRegexp(TypeError, msg, self.resolve, snippet, tmpl)
def test_yaql_bogus_keys(self):
snippet = {'yaql': {'expression': '1 + 3',
'data': 'mustbeamap',
'bogus': ""}}
tmpl = template.Template(hot_newton_tpl_empty)
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
def test_yaql_invalid_syntax(self):
snippet = {'yaql': {'wrong': 'wrong_expr',
'wrong_data': 'mustbeamap'}}
tmpl = template.Template(hot_newton_tpl_empty)
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
def test_yaql_non_map_args(self):
snippet = {'yaql': 'invalid'}
tmpl = template.Template(hot_newton_tpl_empty)
msg = 'Arguments to "yaql" must be a map.'
self.assertRaisesRegexp(TypeError, msg, self.resolve, snippet, tmpl)
def test_yaql_invalid_expression(self):
snippet = {'yaql': {'expression': 'invalid(',
'data': {'var1': [1, 2, 3, 4]}}}
tmpl = template.Template(hot_newton_tpl_empty)
yaql = tmpl.parse(None, snippet)
self.assertRaises(ValueError, function.validate, yaql)
def test_yaql_data_as_function(self):
snippet = {'yaql': {'expression': '$.data.var1.len()',
'data': {
'var1': {'list_join': ['', ['1', '2']]}
}
}}
tmpl = template.Template(hot_newton_tpl_empty)
stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl)
resolved = self.resolve(snippet, tmpl, stack=stack)
self.assertEqual(2, resolved)
def test_repeat(self): def test_repeat(self):
"""Test repeat function.""" """Test repeat function."""
snippet = {'repeat': {'template': 'this is %var%', snippet = {'repeat': {'template': 'this is %var%',

1
requirements.txt
View File

@ -60,3 +60,4 @@ SQLAlchemy<1.1.0,>=1.0.10 # MIT
sqlalchemy-migrate>=0.9.6 # Apache-2.0 sqlalchemy-migrate>=0.9.6 # Apache-2.0
stevedore>=1.10.0 # Apache-2.0 stevedore>=1.10.0 # Apache-2.0
WebOb>=1.2.3 # MIT WebOb>=1.2.3 # MIT
yaql>=1.1.0 # Apache 2.0 License