From e7c51a2529d5fa73441451f01f45b450d367e295 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 20 Sep 2018 07:32:53 +0000 Subject: [PATCH] Series Upgrade Implement the series-upgrade feature allowing to move between Ubuntu series. Change-Id: I13770631becfefb71075a3b0080db2bffc2b268d --- .../contrib/openstack/amulet/utils.py | 3 ++ .../charmhelpers/contrib/openstack/context.py | 4 +++ hooks/charmhelpers/contrib/openstack/utils.py | 30 ++++++++++++++++++- hooks/charmhelpers/core/hookenv.py | 3 +- .../charmhelpers/core/host_factory/ubuntu.py | 1 + hooks/charmhelpers/fetch/__init__.py | 2 ++ hooks/charmhelpers/fetch/bzrurl.py | 4 +-- hooks/charmhelpers/fetch/giturl.py | 4 +-- hooks/charmhelpers/fetch/ubuntu.py | 20 +++++++++++++ hooks/hooks.py | 24 +++++++++++++++ hooks/post-series-upgrade | 1 + hooks/pre-series-upgrade | 1 + tests/charmhelpers/core/hookenv.py | 3 +- .../charmhelpers/core/host_factory/ubuntu.py | 1 + 14 files changed, 94 insertions(+), 7 deletions(-) create mode 120000 hooks/post-series-upgrade create mode 120000 hooks/pre-series-upgrade diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 6637865d..936b4036 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils): cmd, code, output)) amulet.raise_status(amulet.FAIL, msg=msg) + # For mimic ceph osd lspools output + output = output.replace("\n", ",") + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, for pool in str(output).split(','): pool_id_name = pool.split(' ') diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ca913961..3e4e82a7 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-qos', 'default': False, }, + 'enable_nsg_logging': { + 'rel_key': 'enable-nsg-logging', + 'default': False, + }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 24f5b808..ae48d6b4 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([ ('queens', ['2.16.0', '2.17.0']), ('rocky', - ['2.18.0']), + ['2.18.0', '2.19.0']), ]) # >= Liberty version->codename mapping @@ -1733,3 +1733,31 @@ def is_unit_upgrading_set(): return not(not(kv.get('unit-upgrading'))) except Exception: return False + + +def series_upgrade_prepare(pause_unit_helper=None, configs=None): + """ Run common series upgrade prepare tasks. + + :param pause_unit_helper: function: Function to pause unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + set_unit_upgrading() + if pause_unit_helper and configs: + if not is_unit_paused_set(): + pause_unit_helper(configs) + + +def series_upgrade_complete(resume_unit_helper=None, configs=None): + """ Run common series upgrade complete tasks. + + :param resume_unit_helper: function: Function to resume unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + clear_unit_paused() + clear_unit_upgrading() + if configs: + configs.write_all() + if resume_unit_helper: + resume_unit_helper(configs) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 68800074..9abf2a45 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py index 99451b59..a6d375af 100644 --- a/hooks/charmhelpers/core/host_factory/ubuntu.py +++ b/hooks/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', ) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 480a6276..8572d34f 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__ fetch = importlib.import_module(module) filter_installed_packages = fetch.filter_installed_packages +filter_missing_packages = fetch.filter_missing_packages install = fetch.apt_install upgrade = fetch.apt_upgrade update = _fetch_update = fetch.apt_update @@ -96,6 +97,7 @@ if __platform__ == "ubuntu": apt_update = fetch.apt_update apt_upgrade = fetch.apt_upgrade apt_purge = fetch.apt_purge + apt_autoremove = fetch.apt_autoremove apt_mark = fetch.apt_mark apt_hold = fetch.apt_hold apt_unhold = fetch.apt_unhold diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 07cd0293..c4ab3ff1 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call +from subprocess import STDOUT, check_output from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): cmd = ['bzr', 'branch'] cmd += cmd_opts cmd += [source, dest] - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, dest=None, revno=None): url_parts = self.parse_url(source) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4cf21bc2..070ca9bb 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from subprocess import check_call, CalledProcessError +from subprocess import check_output, CalledProcessError, STDOUT from charmhelpers.fetch import ( BaseFetchHandler, UnhandledSource, @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', 'clone', source, dest, '--branch', branch] if depth: cmd.extend(['--depth', depth]) - check_call(cmd) + check_output(cmd, stderr=STDOUT) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 19aa6baf..ec08cbc2 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -189,6 +189,18 @@ def filter_installed_packages(packages): return _pkgs +def filter_missing_packages(packages): + """Return a list of packages that are installed. + + :param packages: list of packages to evaluate. + :returns list: Packages that are installed. + """ + return list( + set(packages) - + set(filter_installed_packages(packages)) + ) + + def apt_cache(in_memory=True, progress=None): """Build and return an apt cache.""" from apt import apt_pkg @@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False): _run_apt_command(cmd, fatal) +def apt_autoremove(purge=True, fatal=False): + """Purge one or more packages.""" + cmd = ['apt-get', '--assume-yes', 'autoremove'] + if purge: + cmd.append('--purge') + _run_apt_command(cmd, fatal) + + def apt_mark(packages, mark, fatal=False): """Flag one or more packages using apt-mark.""" log("Marking {} as {}".format(packages, mark)) diff --git a/hooks/hooks.py b/hooks/hooks.py index ea1fdaec..4f68459f 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -64,6 +64,8 @@ from charmhelpers.contrib.storage.linux.ceph import ( from charmhelpers.contrib.openstack.utils import ( is_unit_paused_set, pausable_restart_on_change as restart_on_change, + series_upgrade_prepare, + series_upgrade_complete, ) from charmhelpers.contrib.hahelpers.cluster import ( get_hacluster_config, @@ -80,6 +82,8 @@ from utils import ( assess_status, setup_keystone_certs, disable_unused_apache_sites, + pause_unit_helper, + resume_unit_helper, ) from charmhelpers.contrib.charmsupport import nrpe from charmhelpers.contrib.hardening.harden import harden @@ -138,6 +142,12 @@ def install(): '/etc/haproxy/haproxy.cfg': ['haproxy']}) @harden() def config_changed(): + # if we are paused, delay doing any config changed hooks. + # It is forced on the resume. + if is_unit_paused_set(): + log("Unit is pause or upgrading. Skipping config_changed", "WARN") + return + install_packages() disable_unused_apache_sites() @@ -383,6 +393,20 @@ def update_status(): log('Updating status.') +@hooks.hook('pre-series-upgrade') +def pre_series_upgrade(): + log("Running prepare series upgrade hook", "INFO") + series_upgrade_prepare( + pause_unit_helper, CONFIGS) + + +@hooks.hook('post-series-upgrade') +def post_series_upgrade(): + log("Running complete series upgrade hook", "INFO") + series_upgrade_complete( + resume_unit_helper, CONFIGS) + + if __name__ == '__main__': try: hooks.execute(sys.argv) diff --git a/hooks/post-series-upgrade b/hooks/post-series-upgrade new file mode 120000 index 00000000..9416ca6a --- /dev/null +++ b/hooks/post-series-upgrade @@ -0,0 +1 @@ +hooks.py \ No newline at end of file diff --git a/hooks/pre-series-upgrade b/hooks/pre-series-upgrade new file mode 120000 index 00000000..9416ca6a --- /dev/null +++ b/hooks/pre-series-upgrade @@ -0,0 +1 @@ +hooks.py \ No newline at end of file diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index 68800074..9abf2a45 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -48,6 +48,7 @@ INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" MARKER = object() +SH_MAX_ARG = 131071 cache = {} @@ -98,7 +99,7 @@ def log(message, level=None): command += ['-l', level] if not isinstance(message, six.string_types): message = repr(message) - command += [message] + command += [message[:SH_MAX_ARG]] # Missing juju-log should not cause failures in unit tests # Send log output to stderr try: diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py index 99451b59..a6d375af 100644 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ b/tests/charmhelpers/core/host_factory/ubuntu.py @@ -21,6 +21,7 @@ UBUNTU_RELEASES = ( 'zesty', 'artful', 'bionic', + 'cosmic', )