From af2e7fd73e2fa643dbe1a0c8cc31d3aa4165720d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Sat, 19 Jan 2019 08:17:10 +0000 Subject: [PATCH] Add custom filters for checking services These filters can be used to capture a lot of the logic that we currently have in 'when' statements, about which services are enabled for a particular host. In order to use these filters, it is necessary to install the kolla_ansible python module, and not just the dependencies listed in requirements.txt. The CI test and quickstart install from source documentation has been updated accordingly. Ansible is not currently in OpenStack global requirements, so for unit tests we avoid a direct dependency on Ansible and provide fakes where necessary. Change-Id: Ib91cac3c28e2b5a834c9746b1d2236a309529556 --- .stestr.conf | 2 +- ansible/filter_plugins/services.py | 22 +++ doc/source/user/quickstart.rst | 8 +- kolla_ansible/exception.py | 24 +++ kolla_ansible/filters.py | 107 +++++++++++ kolla_ansible/tests/__init__.py | 0 kolla_ansible/tests/unit/__init__.py | 0 kolla_ansible/tests/unit/test_filters.py | 173 ++++++++++++++++++ .../notes/jinja-filters-818d5bb97ddc75c6.yaml | 13 ++ tests/run.yml | 8 +- 10 files changed, 348 insertions(+), 9 deletions(-) create mode 100644 ansible/filter_plugins/services.py create mode 100644 kolla_ansible/exception.py create mode 100644 kolla_ansible/filters.py create mode 100644 kolla_ansible/tests/__init__.py create mode 100644 kolla_ansible/tests/unit/__init__.py create mode 100644 kolla_ansible/tests/unit/test_filters.py create mode 100644 releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml diff --git a/.stestr.conf b/.stestr.conf index dcdf16e8d7..3bc54b7516 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,4 +1,4 @@ [DEFAULT] -test_path=./tests +test_path=./ top_dir=./ diff --git a/ansible/filter_plugins/services.py b/ansible/filter_plugins/services.py new file mode 100644 index 0000000000..ed2b831f54 --- /dev/null +++ b/ansible/filter_plugins/services.py @@ -0,0 +1,22 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kolla_ansible import filters + + +class FilterModule(object): + """Service filters.""" + + def filters(self): + return filters.get_filters() diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index df7a058c65..ce942178ca 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -220,15 +220,15 @@ Install Kolla for development .. code-block:: console - pip install -r kolla/requirements.txt - pip install -r kolla-ansible/requirements.txt + pip install ./kolla + pip install ./kolla-ansible If not using a virtual environment: .. code-block:: console - sudo pip install -r kolla/requirements.txt - sudo pip install -r kolla-ansible/requirements.txt + sudo pip install ./kolla + sudo pip install ./kolla-ansible #. Create the ``/etc/kolla`` directory. diff --git a/kolla_ansible/exception.py b/kolla_ansible/exception.py new file mode 100644 index 0000000000..04b670345d --- /dev/null +++ b/kolla_ansible/exception.py @@ -0,0 +1,24 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + from ansible.errors import AnsibleFilterError +except ImportError: + # NOTE(mgoddard): For unit testing we don't depend on Ansible since it is + # not in global requirements. + AnsibleFilterError = Exception + + +class FilterError(AnsibleFilterError): + """Error during execution of a jinja2 filter.""" diff --git a/kolla_ansible/filters.py b/kolla_ansible/filters.py new file mode 100644 index 0000000000..0b238ecc87 --- /dev/null +++ b/kolla_ansible/filters.py @@ -0,0 +1,107 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jinja2 + +from kolla_ansible import exception + + +def _call_bool_filter(context, value): + """Pass a value through the 'bool' filter. + + :param context: Jinja2 Context object. + :param value: Value to pass through bool filter. + :returns: A boolean. + """ + return context.environment.call_filter("bool", value, context=context) + + +@jinja2.contextfilter +def service_enabled(context, service): + """Return whether a service is enabled. + + :param context: Jinja2 Context object. + :param service: Service definition, dict. + :returns: A boolean. + """ + enabled = service.get('enabled') + if enabled is None: + raise exception.FilterError( + "Service definition for '%s' does not have an 'enabled' attribute" + % service.get("container_name", "")) + return _call_bool_filter(context, enabled) + + +@jinja2.contextfilter +def service_mapped_to_host(context, service): + """Return whether a service is mapped to this host. + + There are two ways to describe the service to host mapping. The most common + is via a 'group' attribute, where the service is mapped to all hosts in the + group. The second approach is via a 'host_in_groups' attribute, which is a + boolean expression which should be evaluated for every host. The latter + approach takes precedence over the first. + + :param context: Jinja2 Context object. + :param service: Service definition, dict. + :returns: A boolean. + """ + host_in_groups = service.get("host_in_groups") + if host_in_groups is not None: + return _call_bool_filter(context, host_in_groups) + + group = service.get("group") + if group is not None: + return group in context.get("groups") + + raise exception.FilterError( + "Service definition for '%s' does not have a 'group' or " + "'host_in_groups' attribute" % + service.get("container_name", "")) + + +@jinja2.contextfilter +def service_enabled_and_mapped_to_host(context, service): + """Return whether a service is enabled and mapped to this host. + + :param context: Jinja2 Context object. + :param service: Service definition, dict. + :returns: A boolean. + """ + return (service_enabled(context, service) and + service_mapped_to_host(context, service)) + + +@jinja2.contextfilter +def select_services_enabled_and_mapped_to_host(context, services): + """Select services that are enabled and mapped to this host. + + :param context: Jinja2 Context object. + :param services: Service definitions, dict. + :returns: A dict containing enabled services mapped to this host. + """ + return {service_name: service + for service_name, service in services.items() + if service_enabled_and_mapped_to_host(context, service)} + + +def get_filters(): + return { + "service_enabled": service_enabled, + "service_mapped_to_host": service_mapped_to_host, + "service_enabled_and_mapped_to_host": ( + service_enabled_and_mapped_to_host), + "select_services_enabled_and_mapped_to_host": ( + select_services_enabled_and_mapped_to_host), + } diff --git a/kolla_ansible/tests/__init__.py b/kolla_ansible/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kolla_ansible/tests/unit/__init__.py b/kolla_ansible/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kolla_ansible/tests/unit/test_filters.py b/kolla_ansible/tests/unit/test_filters.py new file mode 100644 index 0000000000..c3314ae8cc --- /dev/null +++ b/kolla_ansible/tests/unit/test_filters.py @@ -0,0 +1,173 @@ +# Copyright (c) 2019 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import jinja2 +import mock + +from kolla_ansible import exception +from kolla_ansible import filters + + +def _to_bool(value): + """Simplified version of the bool filter. + + Avoids having a dependency on Ansible in unit tests. + """ + if value == 'yes': + return True + if value == 'no': + return False + return bool(value) + + +class TestFilters(unittest.TestCase): + + def setUp(self): + # Bandit complains about Jinja2 autoescaping without nosec. + self.env = jinja2.Environment() # nosec + self.env.filters['bool'] = _to_bool + self.context = self._make_context() + + def _make_context(self, parent=None): + if parent is None: + parent = {} + return self.env.context_class( + self.env, parent=parent, name='dummy', blocks={}) + + def test_service_enabled_true(self): + service = { + 'enabled': True + } + self.assertTrue(filters.service_enabled(self.context, service)) + + def test_service_enabled_yes(self): + service = { + 'enabled': 'yes' + } + self.assertTrue(filters.service_enabled(self.context, service)) + + def test_service_enabled_false(self): + service = { + 'enabled': False + } + self.assertFalse(filters.service_enabled(self.context, service)) + + def test_service_enabled_no(self): + service = { + 'enabled': 'no' + } + self.assertFalse(filters.service_enabled(self.context, service)) + + def test_service_enabled_no_attr(self): + service = {} + self.assertRaises(exception.FilterError, + filters.service_enabled, self.context, service) + + def test_service_mapped_to_host_host_in_groups_true(self): + service = { + 'host_in_groups': True + } + self.assertTrue(filters.service_mapped_to_host(self.context, service)) + + def test_service_mapped_to_host_host_in_groups_yes(self): + service = { + 'host_in_groups': 'yes' + } + self.assertTrue(filters.service_mapped_to_host(self.context, service)) + + def test_service_mapped_to_host_host_in_groups_false(self): + service = { + 'host_in_groups': False + } + self.assertFalse(filters.service_mapped_to_host(self.context, service)) + + def test_service_mapped_to_host_host_in_groups_no(self): + service = { + 'host_in_groups': 'no' + } + self.assertFalse(filters.service_mapped_to_host(self.context, service)) + + def test_service_mapped_to_host_in_group(self): + service = { + 'group': 'foo' + } + context = self._make_context({'groups': ['foo', 'bar']}) + self.assertTrue(filters.service_mapped_to_host(context, service)) + + def test_service_mapped_to_host_not_in_group(self): + service = { + 'group': 'foo' + } + context = self._make_context({'groups': ['bar']}) + self.assertFalse(filters.service_mapped_to_host(context, service)) + + def test_service_mapped_to_host_no_attr(self): + service = {} + self.assertRaises(exception.FilterError, + filters.service_mapped_to_host, self.context, + service) + + @mock.patch.object(filters, 'service_enabled') + @mock.patch.object(filters, 'service_mapped_to_host') + def test_service_enabled_and_mapped_to_host(self, mock_mapped, + mock_enabled): + service = {} + mock_enabled.return_value = True + mock_mapped.return_value = True + self.assertTrue(filters.service_enabled_and_mapped_to_host( + self.context, service)) + mock_enabled.assert_called_once_with(self.context, service) + mock_mapped.assert_called_once_with(self.context, service) + + @mock.patch.object(filters, 'service_enabled') + @mock.patch.object(filters, 'service_mapped_to_host') + def test_service_enabled_and_mapped_to_host_disabled(self, mock_mapped, + mock_enabled): + service = {} + mock_enabled.return_value = False + mock_mapped.return_value = True + self.assertFalse(filters.service_enabled_and_mapped_to_host( + self.context, service)) + mock_enabled.assert_called_once_with(self.context, service) + self.assertFalse(mock_mapped.called) + + @mock.patch.object(filters, 'service_enabled') + @mock.patch.object(filters, 'service_mapped_to_host') + def test_service_enabled_and_mapped_to_host_not_mapped(self, mock_mapped, + mock_enabled): + service = {} + mock_enabled.return_value = True + mock_mapped.return_value = False + self.assertFalse(filters.service_enabled_and_mapped_to_host( + self.context, service)) + mock_enabled.assert_called_once_with(self.context, service) + mock_mapped.assert_called_once_with(self.context, service) + + @mock.patch.object(filters, 'service_enabled_and_mapped_to_host') + def test_select_services_enabled_and_mapped_to_host(self, mock_seamth): + services = { + 'foo': object(), + 'bar': object(), + 'baz': object(), + } + mock_seamth.side_effect = lambda _, s: s != services['bar'] + result = filters.select_services_enabled_and_mapped_to_host( + self.context, services) + expected = { + 'foo': services['foo'], + 'baz': services['baz'], + } + self.assertEqual(expected, result) diff --git a/releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml b/releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml new file mode 100644 index 0000000000..50404844cc --- /dev/null +++ b/releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + When installing ``kolla-ansible`` from source, the ``kolla_ansible`` python + module must now be installed in addition to the python dependencies listed + in ``requirements.txt``. This is done via:: + + pip install /path/to/kolla-ansible + + If the git repository is in the current directory, use the following + to avoid installing the package from PyPI:: + + pip install ./kolla-ansible diff --git a/tests/run.yml b/tests/run.yml index 83477800fb..a95956721e 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -133,9 +133,9 @@ dest: ironic-agent.kernel when: scenario == "ironic" - - name: install kolla-ansible requirements + - name: install kolla-ansible pip: - requirements: "{{ kolla_ansible_src_dir }}/requirements.txt" + name: "{{ kolla_ansible_src_dir }}" become: true - name: copy passwords.yml file @@ -311,9 +311,9 @@ when: "{{ is_ceph }}" when: item.when | default(true) - - name: upgrade kolla-ansible requirements + - name: upgrade kolla-ansible pip: - requirements: "{{ kolla_ansible_src_dir }}/requirements.txt" + name: "{{ kolla_ansible_src_dir }}" become: true # Update passwords.yml to include any new passwords added in this