From 6496af1f6759bf53a51a76439aee49e5e263dde5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 8 Nov 2018 19:04:55 +0000 Subject: [PATCH] Add kayobe seed service upgrade command Adds a new command to upgrade containerised services on the seed: kayobe seed service upgrade This is largely the same as the deploy command, since there is no kolla-ansible bifrost-upgrade command yet. A limitation in Bifrost requires us to remove IPA images before upgrading the bifrost container. Change-Id: Ibbe01e972487e69dbb753a67efcf13e3ae0800de Story: 2004308 Task: 27872 --- ansible/seed-service-upgrade-prep.yml | 32 ++++++++ doc/source/upgrading.rst | 78 ++++++++++++++++++- kayobe/cli/commands.py | 35 +++++++++ kayobe/tests/unit/cli/test_commands.py | 45 +++++++++++ ...seed-service-upgrade-71b847e3658a1948.yaml | 5 ++ setup.cfg | 1 + 6 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 ansible/seed-service-upgrade-prep.yml create mode 100644 releasenotes/notes/seed-service-upgrade-71b847e3658a1948.yaml diff --git a/ansible/seed-service-upgrade-prep.yml b/ansible/seed-service-upgrade-prep.yml new file mode 100644 index 000000000..acdb515e7 --- /dev/null +++ b/ansible/seed-service-upgrade-prep.yml @@ -0,0 +1,32 @@ +--- +- name: Prepare for an upgrade of the seed services + hosts: seed + tasks: + # Bifrost fails if IPA images exist with a different checksum. Move them + # out of the way. + - block: + - name: Find IPA deployment images + find: + path: /var/lib/docker/volumes/bifrost_httpboot/_data + patterns: + # Specify filenames individually to avoid movind previously moved + # images. + - ipa.initramfs + - ipa.initramfs.sha256 + - ipa.vmlinuz + - ipa.vmlinuz.sha256 + register: find_result + become: true + + - name: Set a fact about the current time + set_fact: + ipa_extension: "{{ ansible_date_time.iso8601 }}" + + - name: Move old IPA deployment images to make way for new ones + command: mv {{ item.path }} {{ item.path }}.{{ ipa_extension }} + with_items: "{{ find_result.files }}" + loop_control: + label: "{{ item.path }}" + become: true + + when: not ipa_build_images | bool diff --git a/doc/source/upgrading.rst b/doc/source/upgrading.rst index ae62a5880..5324e95a7 100644 --- a/doc/source/upgrading.rst +++ b/doc/source/upgrading.rst @@ -94,7 +94,10 @@ instead perform a targeted upgrade of specific services where necessary. Upgrading the Seed ================== -Currently, upgrading the seed services is not supported. +The seed services are upgraded in two steps. First, new container images +should be obtained either by building them locally or pulling them from an +image registry. Second, the seed services should be replaced with new +containers created from the new container images. Upgrading Host Packages ----------------------- @@ -113,6 +116,25 @@ To only install updates that have been marked security related:: Note that these commands do not affect packages installed in containers, only those installed on the host. +Building Ironic Deployment Images +--------------------------------- + +.. note:: + + It is possible to use prebuilt deployment images. In this case, this step + can be skipped. + +It is possible to use prebuilt deployment images from the `OpenStack hosted +tarballs `_ or another +source. In some cases it may be necessary to build images locally either to +apply local image customisation or to use a downstream version of Ironic Python +Agent (IPA). In order to build IPA images, the ``ipa_build_images`` variable +should be set to ``True``. To build images locally:: + + (kayobe) $ kayobe seed deployment image build + +To overwrite existing images, add the ``--force-rebuild`` argument. + Upgrading Host Services ----------------------- @@ -123,6 +145,58 @@ It may be necessary to upgrade some host services:: Note that this will not perform full configuration of the host, and will instead perform a targeted upgrade of specific services where necessary. +Building Container Images +------------------------- + +.. note:: + + It is possible to use prebuilt container images from an image registry such + as Dockerhub. In this case, this step can be skipped. + +In some cases it may be necessary to build images locally either to apply local +image customisation or to use a downstream version of kolla. To build images +locally:: + + (kayobe) $ kayobe seed container image build + +In order to push images to a registry after they are built, add the ``--push`` +argument. + +Migrating to Ironic Hardware Types +---------------------------------- + +Classic drivers in ironic were `deprecated +`__ +in the Queens release, and `removed +`__ +in the Rocky release. Nodes registered with ironic in Pike and earlier releases +of Bifrost use the classic drivers by default, and will need to be migrated to +use the new hardware types. The `ironic documentation +`__ +provides details for how to do this, but for the default case of the +``agent_ipmitool`` driver the following procedure may be used, replacing +```` with the name of the host to migrate: + +.. code-block:: console + + $ docker exec -it bifrost_deploy bash + (bifrost_deploy) $ export OS_URL=http://localhost:6385 + (bifrost_deploy) $ export OS_TOKEN=fake + (bifrost_deploy) $ openstack baremetal node maintenance set + (bifrost_deploy) $ openstack baremetal node set --driver ipmi + (bifrost_deploy) $ openstack baremetal node maintenance unset + +Upgrading Containerised Services +-------------------------------- + +Containerised seed services may be upgraded by replacing existing containers +with new containers using updated images which have been pulled from +a registry or built locally. + +To upgrade the containerised seed services:: + + (kayobe) $ kayobe seed service upgrade + Upgrading the Overcloud ======================= @@ -178,6 +252,8 @@ should be set to ``True``. To build images locally:: (kayobe) $ kayobe overcloud deployment image build +To overwrite existing images, add the ``--force-rebuild`` argument. + Upgrading Ironic Deployment Images ---------------------------------- diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index b5d59a07b..44472b710 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -512,6 +512,41 @@ class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks) +class SeedServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): + """Upgrade the seed services. + + * Configures kolla-ansible. + * Configures the bifrost service. + * Prepares the bifrost service for an upgrade. + * Deploys the bifrost container using kolla-ansible. + * Builds disk images for the overcloud hosts using Diskimage Builder (DIB). + * Performs a workaround in the overcloud host image to fix resolv.conf. + * Performs a workaround in the overcloud host image to update cloud-init + * Configures ironic inspector introspection rules in the bifrost inspector + service. + * When enabled, configures a Bare Metal Provisioning (BMP) environment for + Dell Force10 switches, hosted by the bifrost dnsmasq and nginx services. + """ + + def take_action(self, parsed_args): + self.app.LOG.debug("Upgrading seed services") + playbooks = _build_playbook_list("kolla-ansible") + self.run_kayobe_playbooks(parsed_args, playbooks, tags="config") + + playbooks = _build_playbook_list( + "kolla-bifrost", + "seed-service-upgrade-prep") + self.run_kayobe_playbooks(parsed_args, playbooks) + self.run_kolla_ansible_seed(parsed_args, "deploy-bifrost") + playbooks = _build_playbook_list( + "overcloud-host-image-workaround-resolv", + "overcloud-host-image-workaround-cloud-init", + "seed-introspection-rules", + "dell-switch-bmp") + self.run_kayobe_playbooks(parsed_args, playbooks) + + class SeedContainerImageBuild(KayobeAnsibleMixin, VaultMixin, Command): """Build the seed container images. diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 1d0b3238a..52c0b9714 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -688,6 +688,51 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_seed") + def test_seed_service_upgrade(self, mock_kolla_run, mock_run): + command = commands.SeedServiceUpgrade(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + ["ansible/kolla-ansible.yml"], + tags="config", + ), + mock.call( + mock.ANY, + [ + "ansible/kolla-bifrost.yml", + "ansible/seed-service-upgrade-prep.yml" + ], + ), + mock.call( + mock.ANY, + [ + "ansible/overcloud-host-image-workaround-resolv.yml", + "ansible/overcloud-host-image-workaround-cloud-init.yml", + "ansible/seed-introspection-rules.yml", + "ansible/dell-switch-bmp.yml", + ], + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + expected_calls = [ + mock.call( + mock.ANY, + "deploy-bifrost", + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbook") def test_overcloud_inventory_discover(self, mock_run): diff --git a/releasenotes/notes/seed-service-upgrade-71b847e3658a1948.yaml b/releasenotes/notes/seed-service-upgrade-71b847e3658a1948.yaml new file mode 100644 index 000000000..e0f0469b1 --- /dev/null +++ b/releasenotes/notes/seed-service-upgrade-71b847e3658a1948.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a new command to upgrade containerised seed services, ``kayobe seed + service upgrade``. diff --git a/setup.cfg b/setup.cfg index 142f1e831..00dd002cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,5 +69,6 @@ kayobe.cli= seed_hypervisor_host_configure = kayobe.cli.commands:SeedHypervisorHostConfigure seed_hypervisor_host_upgrade = kayobe.cli.commands:SeedHypervisorHostUpgrade seed_service_deploy = kayobe.cli.commands:SeedServiceDeploy + seed_service_upgrade = kayobe.cli.commands:SeedServiceUpgrade seed_vm_deprovision = kayobe.cli.commands:SeedVMDeprovision seed_vm_provision = kayobe.cli.commands:SeedVMProvision