diff --git a/.gitignore b/.gitignore index ee1126c..925e487 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ tests/cirros-*-disk.img func-results.json __pycache__ .stestr/ +tests/cirros* diff --git a/actions.yaml b/actions.yaml index 04624d7..1aad31f 100644 --- a/actions.yaml +++ b/actions.yaml @@ -5,3 +5,14 @@ openstack-upgrade: domain-setup: description: Setup the keystone domains, roles and user required for Heat to operate. Only required for OpenStack >= Kilo. +pause: + description: | + Pause heat services. + If the heat deployment is clustered using the hacluster charm, the + corresponding hacluster unit on the node must first be paused as well. + Not doing so may lead to an interruption of service. +resume: + description: | + Resume heat services. + If the heat deployment is clustered using the hacluster charm, the + corresponding hacluster unit on the node must be resumed as well. diff --git a/actions/actions.py b/actions/actions.py new file mode 100755 index 0000000..e932b1c --- /dev/null +++ b/actions/actions.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os + +_path = os.path.dirname(os.path.realpath(__file__)) +_parent = os.path.abspath(os.path.join(_path, "..")) +_hooks = os.path.abspath(os.path.join(_parent, "hooks")) + + +def _add_path(path): + if path not in sys.path: + sys.path.insert(1, path) + + +_add_path(_parent) +_add_path(_hooks) + + +from charmhelpers.core.hookenv import action_fail + +sys.path.append('hooks/') + +from heat_utils import ( + pause_unit_helper, + resume_unit_helper, + register_configs, +) + + +def pause(args): + """Pause all the Glance services. + + @raises Exception if any services fail to stop + """ + pause_unit_helper(register_configs()) + + +def resume(args): + """Resume all the Glance services. + + @raises Exception if any services fail to start + """ + resume_unit_helper(register_configs()) + + +# A dictionary of all the defined actions to callables (which take +# parsed arguments). +ACTIONS = {"pause": pause, "resume": resume} + + +def main(args): + action_name = os.path.basename(args[0]) + try: + action = ACTIONS[action_name] + except KeyError: + return "Action %s undefined" % action_name + else: + try: + action(args) + except Exception as e: + action_fail(str(e)) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/actions/pause b/actions/pause new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/actions/pause @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/actions/resume b/actions/resume new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/actions/resume @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/hooks/heat_relations.py b/hooks/heat_relations.py index fc9bbca..547805a 100755 --- a/hooks/heat_relations.py +++ b/hooks/heat_relations.py @@ -64,9 +64,9 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import ( configure_installation_source, openstack_upgrade_available, - set_os_workload_status, sync_db_with_multi_ipv6_addresses, - os_application_version_set, + series_upgrade_prepare, + series_upgrade_complete, ) from charmhelpers.contrib.openstack.ha.utils import ( @@ -88,9 +88,10 @@ from heat_utils import ( register_configs, CLUSTER_RES, HEAT_CONF, - REQUIRED_INTERFACES, setup_ipv6, - VERSION_PACKAGE, + pause_unit_helper, + resume_unit_helper, + assess_status, ) from heat_context import ( @@ -456,13 +457,26 @@ def certs_changed(relation_id=None, unit=None): configure_https() +@hooks.hook('pre-series-upgrade') +def pre_series_upgrade(): + log("Running prepare series upgrade hook", "INFO") + series_upgrade_prepare( + pause_unit_helper, CONFIGS) + + +@hooks.hook('post-series-upgrade') +def post_series_upgrade(): + log("Running complete series upgrade hook", "INFO") + series_upgrade_complete( + resume_unit_helper, CONFIGS) + + def main(): try: hooks.execute(sys.argv) except UnregisteredHookError as e: log('Unknown hook {} - skipping.'.format(e)) - set_os_workload_status(CONFIGS, REQUIRED_INTERFACES) - os_application_version_set(VERSION_PACKAGE) + assess_status(CONFIGS) if __name__ == '__main__': diff --git a/hooks/heat_utils.py b/hooks/heat_utils.py index a006c1b..77cc3e2 100644 --- a/hooks/heat_utils.py +++ b/hooks/heat_utils.py @@ -14,6 +14,7 @@ import os +from copy import deepcopy from collections import OrderedDict from subprocess import check_call @@ -26,8 +27,17 @@ from charmhelpers.contrib.openstack.utils import ( token_cache_pkgs, enable_memcache, CompareOpenStackReleases, + os_application_version_set, + make_assess_status_func, + pause_unit, + resume_unit, ) +from charmhelpers.contrib.hahelpers.cluster import ( + get_hacluster_config, +) + + from charmhelpers.fetch import ( add_source, apt_install, @@ -41,6 +51,7 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( log, config, + relation_ids, ) from charmhelpers.core.host import ( @@ -151,31 +162,41 @@ CONFIG_FILES = OrderedDict([ 'services': [] }), (MEMCACHED_CONF, { - 'hook_contexts': [context.MemcacheContext()], + 'contexts': [context.MemcacheContext()], 'services': ['memcached'], }), ]) -def register_configs(): - release = os_release('heat-common') - configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, - openstack_release=release) - - confs = [HEAT_CONF, HEAT_API_PASTE, HAPROXY_CONF, ADMIN_OPENRC] - for conf in confs: - configs.register(conf, CONFIG_FILES[conf]['contexts']) +def resource_map(release=None): + """ + Dynamically generate a map of resources that will be managed for a single + hook execution. + """ + _release = release or os_release('heat-common', base='icehouse') + _resource_map = deepcopy(CONFIG_FILES) if os.path.exists('/etc/apache2/conf-available'): - configs.register(HTTPS_APACHE_24_CONF, - CONFIG_FILES[HTTPS_APACHE_24_CONF]['contexts']) + _resource_map.pop(HTTPS_APACHE_CONF) else: - configs.register(HTTPS_APACHE_CONF, - CONFIG_FILES[HTTPS_APACHE_CONF]['contexts']) + _resource_map.pop(HTTPS_APACHE_24_CONF) - if enable_memcache(release=release): - configs.register(MEMCACHED_CONF, - CONFIG_FILES[MEMCACHED_CONF]['hook_contexts']) + if not enable_memcache(release=_release): + _resource_map.pop(MEMCACHED_CONF) + + return _resource_map + + +def register_configs(release=None): + """Register config files with their respective contexts. + Regstration of some configs may not be required depending on + existing of certain relations. + """ + release = release or os_release('heat-common', base='icehouse') + configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, + openstack_release=release) + for cfg, rscs in resource_map(release).items(): + configs.register(cfg, rscs['contexts']) return configs @@ -249,22 +270,15 @@ def do_openstack_upgrade(configs): def restart_map(): - """Restarts on config change. - - Determine the correct resource map to be passed to + '''Determine the correct resource map to be passed to charmhelpers.core.restart_on_change() based on the services configured. :returns: dict: A dictionary mapping config file to lists of services - that should be restarted when file changes. - """ - _map = [] - for f, ctxt in CONFIG_FILES.items(): - svcs = [] - for svc in ctxt['services']: - svcs.append(svc) - if svcs: - _map.append((f, svcs)) - return OrderedDict(_map) + that should be restarted when file changes. + ''' + return OrderedDict([(cfg, v['services']) + for cfg, v in resource_map().items() + if v['services']]) def services(): @@ -297,3 +311,113 @@ def setup_ipv6(): 'main') apt_update() apt_install('haproxy/trusty-backports', fatal=True) + + +def check_optional_relations(configs): + """Check that if we have a relation_id for high availability that we can + get the hacluster config. If we can't then we are blocked. + + This function is called from assess_status/set_os_workload_status as the + charm_func and needs to return either None, None if there is no problem or + the status, message if there is a problem. + + :param configs: an OSConfigRender() instance. + :return 2-tuple: (string, string) = (status, message) + """ + if relation_ids('ha'): + try: + get_hacluster_config() + except: + return ('blocked', + 'hacluster missing configuration: ' + 'vip, vip_iface, vip_cidr') + # return 'unknown' as the lowest priority to not clobber an existing + # status. + return "unknown", "" + + +def get_optional_interfaces(): + """Return the optional interfaces that should be checked if the relavent + relations have appeared. + + :returns: {general_interface: [specific_int1, specific_int2, ...], ...} + """ + optional_interfaces = {} + if relation_ids('ha'): + optional_interfaces['ha'] = ['cluster'] + + return optional_interfaces + + +def assess_status(configs): + """Assess status of current unit + Decides what the state of the unit should be based on the current + configuration. + SIDE EFFECT: calls set_os_workload_status(...) which sets the workload + status of the unit. + Also calls status_set(...) directly if paused state isn't complete. + @param configs: a templating.OSConfigRenderer() object + @returns None - this function is executed for its side-effect + """ + assess_status_func(configs)() + os_application_version_set(VERSION_PACKAGE) + + +def assess_status_func(configs): + """Helper function to create the function that will assess_status() for + the unit. + Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to + create the appropriate status function and then returns it. + Used directly by assess_status() and also for pausing and resuming + the unit. + + NOTE: REQUIRED_INTERFACES is augmented with the optional interfaces + depending on the current config before being passed to the + make_assess_status_func() function. + + NOTE(ajkavanagh) ports are not checked due to race hazards with services + that don't behave sychronously w.r.t their service scripts. e.g. + apache2. + @param configs: a templating.OSConfigRenderer() object + @return f() -> None : a function that assesses the unit's workload status + """ + required_interfaces = REQUIRED_INTERFACES.copy() + required_interfaces.update(get_optional_interfaces()) + return make_assess_status_func( + configs, required_interfaces, + charm_func=check_optional_relations, + services=services(), ports=None) + + +def pause_unit_helper(configs): + """Helper function to pause a unit, and then call assess_status(...) in + effect, so that the status is correctly updated. + Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work. + @param configs: a templating.OSConfigRenderer() object + @returns None - this function is executed for its side-effect + """ + _pause_resume_helper(pause_unit, configs) + + +def resume_unit_helper(configs): + """Helper function to resume a unit, and then call assess_status(...) in + effect, so that the status is correctly updated. + Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work. + @param configs: a templating.OSConfigRenderer() object + @returns None - this function is executed for its side-effect + """ + _pause_resume_helper(resume_unit, configs) + + +def _pause_resume_helper(f, configs): + """Helper function that uses the make_assess_status_func(...) from + charmhelpers.contrib.openstack.utils to create an assess_status(...) + function that can be used with the pause/resume of the unit + @param f: the function to be used with the assess_status(...) function + @returns None - this function is executed for its side-effect + """ + # TODO(ajkavanagh) - ports= has been left off because of the race hazard + # that exists due to service_start() + f(assess_status_func(configs), + services=services(), + ports=None) diff --git a/hooks/post-series-upgrade b/hooks/post-series-upgrade new file mode 120000 index 0000000..ab98840 --- /dev/null +++ b/hooks/post-series-upgrade @@ -0,0 +1 @@ +heat_relations.py \ No newline at end of file diff --git a/hooks/pre-series-upgrade b/hooks/pre-series-upgrade new file mode 120000 index 0000000..ab98840 --- /dev/null +++ b/hooks/pre-series-upgrade @@ -0,0 +1 @@ +heat_relations.py \ No newline at end of file diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 04d7e98..5f6b5ae 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -719,3 +719,27 @@ class HeatBasicDeployment(OpenStackAmuletDeployment): sleep_time = 0 self.d.configure(juju_service, set_default) + + def test_901_pause_resume(self): + """Test pause and resume actions.""" + u.log.debug('Checking pause and resume actions...') + unit = self.d.sentry['heat'][0] + unit_name = unit.info['unit_name'] + + u.log.debug('Checking for active status on {}'.format(unit_name)) + assert u.status_get(unit)[0] == "active" + + u.log.debug('Running pause action on {}'.format(unit_name)) + action_id = u.run_action(unit, "pause") + u.log.debug('Waiting on action {}'.format(action_id)) + assert u.wait_on_action(action_id), "Pause action failed." + u.log.debug('Checking for maintenance status on {}'.format(unit_name)) + assert u.status_get(unit)[0] == "maintenance" + + u.log.debug('Running resume action on {}'.format(unit_name)) + action_id = u.run_action(unit, "resume") + u.log.debug('Waiting on action {}'.format(action_id)) + assert u.wait_on_action(action_id), "Resume action failed." + u.log.debug('Checking for active status on {}'.format(unit_name)) + assert u.status_get(unit)[0] == "active" + u.log.debug('OK') diff --git a/unit_tests/test_actions.py b/unit_tests/test_actions.py new file mode 100644 index 0000000..5c7e084 --- /dev/null +++ b/unit_tests/test_actions.py @@ -0,0 +1,80 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mock + +from test_utils import CharmTestCase + +os.environ['JUJU_UNIT_NAME'] = 'heat' +with mock.patch('heat_utils.register_configs') as configs: + configs.return_value = 'test-config' + import actions + + +class PauseTestCase(CharmTestCase): + + def setUp(self): + super(PauseTestCase, self).setUp( + actions, ["pause_unit_helper"]) + + def test_pauses_services(self): + actions.pause([]) + self.pause_unit_helper.assert_called_once_with('test-config') + + +class ResumeTestCase(CharmTestCase): + + def setUp(self): + super(ResumeTestCase, self).setUp( + actions, ["resume_unit_helper"]) + + def test_pauses_services(self): + actions.resume([]) + self.resume_unit_helper.assert_called_once_with('test-config') + + +class MainTestCase(CharmTestCase): + + def setUp(self): + super(MainTestCase, self).setUp(actions, ["action_fail"]) + + def test_invokes_action(self): + dummy_calls = [] + + def dummy_action(args): + dummy_calls.append(True) + + with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}): + actions.main(["foo"]) + self.assertEqual(dummy_calls, [True]) + + def test_unknown_action(self): + """Unknown actions aren't a traceback.""" + exit_string = actions.main(["foo"]) + self.assertEqual("Action foo undefined", exit_string) + + def test_failing_action(self): + """Actions which traceback trigger action_fail() calls.""" + dummy_calls = [] + + self.action_fail.side_effect = dummy_calls.append + + def dummy_action(args): + raise ValueError("uh oh") + + with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}): + actions.main(["foo"]) + self.assertEqual(dummy_calls, ["uh oh"]) diff --git a/unit_tests/test_actions_openstack_upgrade.py b/unit_tests/test_actions_openstack_upgrade.py index 54a13e2..6197d57 100644 --- a/unit_tests/test_actions_openstack_upgrade.py +++ b/unit_tests/test_actions_openstack_upgrade.py @@ -28,7 +28,8 @@ mock_apt.apt_pkg = MagicMock() with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: lambda *args, **kwargs: f(*args, **kwargs)) - with patch('heat_utils.register_configs') as register_configs: + with patch('heat_utils.register_configs') as register_configs, \ + patch('heat_utils.resource_map') as resource_map: import openstack_upgrade from test_utils import ( diff --git a/unit_tests/test_heat_utils.py b/unit_tests/test_heat_utils.py index 9589540..8f117ba 100644 --- a/unit_tests/test_heat_utils.py +++ b/unit_tests/test_heat_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from copy import deepcopy from collections import OrderedDict from mock import patch, MagicMock, call from test_utils import CharmTestCase @@ -39,6 +40,7 @@ TO_PATCH = [ 'service_stop', 'token_cache_pkgs', 'enable_memcache', + 'os' ] @@ -101,7 +103,24 @@ class HeatUtilsTests(CharmTestCase): ['python-heat', 'python-memcache']) def test_restart_map(self): - self.assertEqual(RESTART_MAP, utils.restart_map()) + # Icehouse + self.os_release.return_value = "icehouse" + self.enable_memcache.return_value = False + self.os.path.exists.return_value = False + _restart_map = deepcopy(RESTART_MAP) + _restart_map.pop( + "/etc/apache2/sites-available/openstack_https_frontend.conf") + _restart_map.pop("/etc/memcached.conf") + self.assertEqual(_restart_map, utils.restart_map()) + + # Mitaka + self.os_release.return_value = "mitaka" + self.enable_memcache.return_value = True + self.os.path.exists.return_value = True + _restart_map = deepcopy(RESTART_MAP) + _restart_map.pop( + "/etc/apache2/sites-available/openstack_https_frontend") + self.assertEqual(_restart_map, utils.restart_map()) def test_openstack_upgrade(self): self.config.side_effect = None