From 6e7b3be2b87d3a8b3d07efcf83cace7f9ab39448 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Tue, 14 Mar 2017 15:34:31 -0400 Subject: [PATCH] Add a make_url intrinsic function A large proportion of uses of the str_replace function is to build URLs out of various components from various sources. This is invariably brittle, with a failure to escape special characters, deal with IPv6 addresses, and so on. The make_url function provides a both a tidier interface and correct handling of these edge cases. Change-Id: I61b6dff01cd509b3d1c54bca118632c276569f4e --- doc/source/template_guide/hot_spec.rst | 48 +++- heat/engine/hot/functions.py | 117 +++++++++ heat/engine/hot/template.py | 3 + heat/tests/test_hot.py | 225 ++++++++++++++++++ .../make_url-function-d76737adb1e54801.yaml | 5 + 5 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/make_url-function-d76737adb1e54801.yaml diff --git a/doc/source/template_guide/hot_spec.rst b/doc/source/template_guide/hot_spec.rst index 8f099dac3b..68429511f9 100644 --- a/doc/source/template_guide/hot_spec.rst +++ b/doc/source/template_guide/hot_spec.rst @@ -293,7 +293,8 @@ for the ``heat_template_version`` key: ------------------- The key with value ``2017-09-01`` or ``pike`` indicates that the YAML document is a HOT template and it may contain features added and/or removed - up until the Pike release. The complete list of supported functions is:: + up until the Pike release. This version adds the ``make_url`` function for + assembling URLs. The complete list of supported functions is:: digest filter @@ -302,6 +303,7 @@ for the ``heat_template_version`` key: get_param get_resource list_join + make_url map_merge map_replace repeat @@ -1845,3 +1847,47 @@ For example - {get_param: list_param} output_list will be evaluated to [1, 2]. + +make_url +-------- + +The ``make_url`` function builds URLs. + +The syntax of the ``make_url`` function is + +.. code-block:: yaml + + make_url: + scheme: + username: + password: + host: + port: + path: + query: + : + : + fragment: + + +All parameters are optional. + +For example + +.. code-block:: yaml + + outputs: + server_url: + value: + make_url: + scheme: http + host: {get_attr: [server, networks, , 0]} + port: 8080 + path: /hello + query: + recipient: world + fragment: greeting + +``server_url`` will be evaluated to a URL in the form:: + + http://[]:8080/hello?recipient=world#greeting diff --git a/heat/engine/hot/functions.py b/heat/engine/hot/functions.py index 970e912d84..07d2d68362 100644 --- a/heat/engine/hot/functions.py +++ b/heat/engine/hot/functions.py @@ -18,6 +18,7 @@ import itertools from oslo_config import cfg from oslo_serialization import jsonutils import six +from six.moves.urllib import parse as urlparse import yaql from yaql.language import exceptions @@ -1320,3 +1321,119 @@ class Filter(function.Function): raise TypeError( _('"%(fn)s" filters a list of values') % self.fn_name) return [i for i in sequence if i not in values] + + +class MakeURL(function.Function): + """A function for performing substitutions on maps. + + Takes the form:: + + make_url: + scheme: + username: + password: + host: + port: + path: + query: + : + fragment: + + And resolves to a correctly-escaped URL constructed from the various + components. + """ + + _ARG_KEYS = ( + SCHEME, USERNAME, PASSWORD, HOST, PORT, + PATH, QUERY, FRAGMENT, + ) = ( + 'scheme', 'username', 'password', 'host', 'port', + 'path', 'query', 'fragment', + ) + + def _check_args(self, args): + for arg in self._ARG_KEYS: + if arg in args: + if arg == self.QUERY: + if not isinstance(args[arg], (function.Function, + collections.Mapping)): + raise TypeError(_('The "%(arg)s" argument to ' + '"(fn_name)%s" must be a map') % + {'arg': arg, + 'fn_name': self.fn_name}) + return + elif arg == self.PORT: + port = args[arg] + if not isinstance(port, function.Function): + if not isinstance(port, six.integer_types): + try: + port = int(port) + except ValueError: + raise ValueError(_('Invalid URL port "%s"') % + port) + if not (0 < port <= 65535): + raise ValueError(_('Invalid URL port %d') % port) + else: + if not isinstance(args[arg], (function.Function, + six.string_types)): + raise TypeError(_('The "%(arg)s" argument to ' + '"(fn_name)%s" must be a string') % + {'arg': arg, + 'fn_name': self.fn_name}) + + def validate(self): + super(MakeURL, self).validate() + + if not isinstance(self.args, collections.Mapping): + raise TypeError(_('The arguments to "%s" must ' + 'be a map') % self.fn_name) + + invalid_keys = set(self.args) - set(self._ARG_KEYS) + if invalid_keys: + raise ValueError(_('Invalid arguments to "%(fn)s": %(args)s') % + {'fn': self.fn_name, + 'args': ', '.join(invalid_keys)}) + + self._check_args(self.args) + + def result(self): + args = function.resolve(self.args) + self._check_args(args) + + scheme = args.get(self.SCHEME, '') + if ':' in scheme: + raise ValueError(_('URL "%s" should not contain \':\'') % + self.SCHEME) + + def netloc(): + username = urlparse.quote(args.get(self.USERNAME, ''), safe='') + password = urlparse.quote(args.get(self.PASSWORD, ''), safe='') + if username or password: + yield username + if password: + yield ':' + yield password + yield '@' + + host = args.get(self.HOST, '') + if host.startswith('[') and host.endswith(']'): + host = host[1:-1] + host = urlparse.quote(host, safe=':') + if ':' in host: + host = '[%s]' % host + yield host + + port = args.get(self.PORT, '') + if port: + yield ':' + yield six.text_type(port) + + path = urlparse.quote(args.get(self.PATH, '')) + + query_dict = args.get(self.QUERY, {}) + query = urlparse.urlencode(query_dict) + + fragment = urlparse.quote(args.get(self.FRAGMENT, '')) + + return urlparse.urlunsplit((scheme, ''.join(netloc()), + path, query, fragment)) diff --git a/heat/engine/hot/template.py b/heat/engine/hot/template.py index 72b990bdb1..ce9de81e18 100644 --- a/heat/engine/hot/template.py +++ b/heat/engine/hot/template.py @@ -588,6 +588,9 @@ class HOTemplate20170901(HOTemplate20170224): 'filter': hot_funcs.Filter, 'str_replace_strict': hot_funcs.ReplaceJsonStrict, + # functions added in 2017-09-01 + 'make_url': hot_funcs.MakeURL, + # functions removed from 2015-10-15 'Fn::Select': hot_funcs.Removed, diff --git a/heat/tests/test_hot.py b/heat/tests/test_hot.py index 23acebd8be..0cdf5cb30a 100644 --- a/heat/tests/test_hot.py +++ b/heat/tests/test_hot.py @@ -1810,6 +1810,231 @@ conditions: stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl) self.assertRaises(TypeError, self.resolve, snippet, tmpl, stack=stack) + def test_make_url_basic(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': 'example.com', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + func = tmpl.parse(None, snippet) + function.validate(func) + resolved = function.resolve(func) + + self.assertEqual('http://example.com/foo/bar', + resolved) + + def test_make_url_ipv6(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': '::1', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('http://[::1]/foo/bar', + resolved) + + def test_make_url_ipv6_ready(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': '[::1]', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('http://[::1]/foo/bar', + resolved) + + def test_make_url_port_string(self): + snippet = { + 'make_url': { + 'scheme': 'https', + 'host': 'example.com', + 'port': '80', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('https://example.com:80/foo/bar', + resolved) + + def test_make_url_port_int(self): + snippet = { + 'make_url': { + 'scheme': 'https', + 'host': 'example.com', + 'port': 80, + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('https://example.com:80/foo/bar', + resolved) + + def test_make_url_port_invalid_high(self): + snippet = { + 'make_url': { + 'scheme': 'https', + 'host': 'example.com', + 'port': 100000, + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + self.assertRaises(ValueError, self.resolve, snippet, tmpl) + + def test_make_url_port_invalid_low(self): + snippet = { + 'make_url': { + 'scheme': 'https', + 'host': 'example.com', + 'port': '0', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + self.assertRaises(ValueError, self.resolve, snippet, tmpl) + + def test_make_url_port_invalid_string(self): + snippet = { + 'make_url': { + 'scheme': 'https', + 'host': 'example.com', + 'port': '1.1', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + self.assertRaises(ValueError, self.resolve, snippet, tmpl) + + def test_make_url_username(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'username': 'wibble', + 'host': 'example.com', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('http://wibble@example.com/foo/bar', + resolved) + + def test_make_url_username_password(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'username': 'wibble', + 'password': 'blarg', + 'host': 'example.com', + 'path': '/foo/bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('http://wibble:blarg@example.com/foo/bar', + resolved) + + def test_make_url_query(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': 'example.com', + 'path': '/foo/?bar', + 'query': { + 'foo': 'bar&baz', + 'blarg': 'wib=ble', + }, + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertIn(resolved, + ['http://example.com/foo/%3Fbar' + '?foo=bar%26baz&blarg=wib%3Dble', + 'http://example.com/foo/%3Fbar' + '?blarg=wib%3Dble&foo=bar%26baz']) + + def test_make_url_fragment(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': 'example.com', + 'path': 'foo/bar', + 'fragment': 'baz' + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('http://example.com/foo/bar#baz', + resolved) + + def test_make_url_file(self): + snippet = { + 'make_url': { + 'scheme': 'file', + 'path': 'foo/bar' + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('file:///foo/bar', + resolved) + + def test_make_url_file_leading_slash(self): + snippet = { + 'make_url': { + 'scheme': 'file', + 'path': '/foo/bar' + } + } + tmpl = template.Template(hot_pike_tpl_empty) + resolved = self.resolve(snippet, tmpl) + + self.assertEqual('file:///foo/bar', + resolved) + + def test_make_url_bad_args_type(self): + snippet = { + 'make_url': 'http://example.com/foo/bar' + } + tmpl = template.Template(hot_pike_tpl_empty) + func = tmpl.parse(None, snippet) + self.assertRaises(exception.StackValidationFailed, function.validate, + func) + + def test_make_url_invalid_key(self): + snippet = { + 'make_url': { + 'scheme': 'http', + 'host': 'example.com', + 'foo': 'bar', + } + } + tmpl = template.Template(hot_pike_tpl_empty) + func = tmpl.parse(None, snippet) + self.assertRaises(exception.StackValidationFailed, function.validate, + func) + def test_depends_condition(self): hot_tpl = template_format.parse(''' heat_template_version: 2016-10-14 diff --git a/releasenotes/notes/make_url-function-d76737adb1e54801.yaml b/releasenotes/notes/make_url-function-d76737adb1e54801.yaml new file mode 100644 index 0000000000..6e52e7d7ee --- /dev/null +++ b/releasenotes/notes/make_url-function-d76737adb1e54801.yaml @@ -0,0 +1,5 @@ +--- +features: + - The Pike version of HOT (2017-09-01) adds a make_url function to simplify + combining data from different sources into a URL with correct handling for + escaping and IPv6 addresses.