From 7f7d36738560c4c2464f9dd175b93955bc0b0832 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 7 Dec 2017 19:01:18 +0000 Subject: [PATCH 1/7] Support configuration of a remote virtualenv for kolla-ansible --- ansible/group_vars/all/kolla | 4 ++++ ansible/roles/kolla-ansible/defaults/main.yml | 4 ++++ ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 | 5 +++++ ansible/roles/kolla-ansible/templates/seed.j2 | 4 ++++ etc/kayobe/kolla.yml | 4 ++++ 5 files changed, 21 insertions(+) diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index 48c044e12..ac2703114 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -250,6 +250,10 @@ kolla_overcloud_inventory_kolla_top_level_groups: ############################################################################### # Kolla-ansible configuration. +# Virtualenv directory where Kolla-ansible's ansible modules will execute +# remotely on the target nodes. If None, no virtualenv will be used. +kolla_ansible_target_venv: "{{ virtualenv_path ~ '/kolla-ansible' }}" + # Password to use to encrypt the kolla-ansible passwords.yml file. kolla_ansible_vault_password: "{{ lookup('env', 'KAYOBE_VAULT_PASSWORD') | default }}" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 6d40e4f4a..8aa472f0d 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -15,6 +15,10 @@ kolla_ansible_source_version: # Virtualenv directory where Kolla-ansible will be installed. kolla_ansible_venv: "{{ ansible_env['PWD'] }}/kolla-venv" +# Virtualenv directory where Kolla-ansible's ansible modules will execute +# remotely on the target nodes. If None, no virtualenv will be used. +kolla_ansible_target_venv: + # Password to use to encrypt the passwords.yml file. kolla_ansible_vault_password: diff --git a/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 b/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 index d540371c5..8458f2a3a 100644 --- a/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 +++ b/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 @@ -30,6 +30,11 @@ [overcloud:vars] ansible_user=kolla ansible_become=true +{% if kolla_ansible_target_venv is not none %} +# Execute ansible modules on the remote target hosts using a virtualenv. +ansible_python_interpreter={{ kolla_ansible_target_venv }}/bin/python +{% endif %} + {% for kolla_group, kolla_group_config in kolla_overcloud_inventory_top_level_group_map.items() %} {% if 'groups' in kolla_group_config %} diff --git a/ansible/roles/kolla-ansible/templates/seed.j2 b/ansible/roles/kolla-ansible/templates/seed.j2 index 2d0e80ab6..20f826694 100644 --- a/ansible/roles/kolla-ansible/templates/seed.j2 +++ b/ansible/roles/kolla-ansible/templates/seed.j2 @@ -7,6 +7,10 @@ [seed:vars] ansible_user=kolla +{% if kolla_ansible_target_venv is not none %} +# Execute ansible modules on the remote target hosts using a virtualenv. +ansible_python_interpreter={{ kolla_ansible_target_venv }}/bin/python +{% endif %} [baremetal:children] seed diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index c27a199d4..ed3baeb09 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -131,6 +131,10 @@ ############################################################################### # Kolla-ansible configuration. +# Virtualenv directory where Kolla-ansible's ansible modules will execute +# remotely on the target nodes. If None, no virtualenv will be used. +#kolla_ansible_target_venv: + # Whether TLS is enabled for the external API endpoints. #kolla_enable_tls_external: From 3620be7c082ffd3da35c5cc90490cdf7906f0bd7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 7 Dec 2017 19:02:37 +0000 Subject: [PATCH 2/7] Support configuration of a remote virtualenv for kayobe --- ansible/kayobe-ansible-user.yml | 3 ++ ansible/kayobe-target-venv.yml | 52 ++++++++++++++++++++++++ ansible/overcloud-docker-sdk-upgrade.yml | 14 ++++++- ansible/roles/docker/tasks/main.yml | 15 +++++++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 ansible/kayobe-target-venv.yml diff --git a/ansible/kayobe-ansible-user.yml b/ansible/kayobe-ansible-user.yml index 85dfb7930..1ef76f82d 100644 --- a/ansible/kayobe-ansible-user.yml +++ b/ansible/kayobe-ansible-user.yml @@ -3,6 +3,9 @@ hosts: seed:overcloud vars: ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python roles: - role: singleplatform-eng.users users: diff --git a/ansible/kayobe-target-venv.yml b/ansible/kayobe-target-venv.yml new file mode 100644 index 000000000..262e4ca04 --- /dev/null +++ b/ansible/kayobe-target-venv.yml @@ -0,0 +1,52 @@ +--- +# Create a virtualenv for ansible modules to use on the remote target systems +# when running kayobe. + +- name: Ensure a virtualenv exists for kayobe + hosts: seed:seed-hypervisor:overcloud + gather_facts: False + tasks: + - name: Set a fact about the kayobe target virtualenv + set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin') + - not ansible_python_interpreter.startswith('/usr/bin') + + - block: + # This will cause ansible to use the system python interpreter. + - name: Deactivate the virtualenv + include_role: + name: deactivate-virtualenv + + - name: Ensure the python-virtualenv package is installed + package: + name: python-virtualenv + state: installed + become: True + + - name: Ensure kayobe virtualenv directory exists + file: + path: "{{ virtualenv }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0700 + become: True + + - name: Ensure kayobe virtualenv has the latest version of pip installed + pip: + name: pip + state: latest + virtualenv: "{{ virtualenv }}" + # Site packages are required for using the yum and selinux python + # modules, which are not available via PyPI. + virtualenv_site_packages: True + + - name: Activate the virtualenv + include_role: + name: activate-virtualenv + vars: + activate_virtualenv_path: "{{ virtualenv }}" + when: virtualenv is defined diff --git a/ansible/overcloud-docker-sdk-upgrade.yml b/ansible/overcloud-docker-sdk-upgrade.yml index c52138a86..8ce71f172 100644 --- a/ansible/overcloud-docker-sdk-upgrade.yml +++ b/ansible/overcloud-docker-sdk-upgrade.yml @@ -5,14 +5,24 @@ # Docker renamed their python SDK from docker-py to docker in the 2.0.0 # release, and also broke backwards compatibility. Kolla-ansible requires # docker, so ensure it is installed. + - name: Set a fact about the virtualenv on the remote system + set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + - name: Ensure legacy docker-py python package is uninstalled pip: name: docker-py state: absent - become: True + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" - name: Ensure docker SDK for python is installed pip: name: docker state: latest - become: True + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index c79a2d98d..374208632 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -6,6 +6,21 @@ include 'devicemapper' and 'overlay'. when: docker_storage_driver not in ['devicemapper', 'overlay'] +- name: Set a fact about the virtualenv on the remote system + set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + +- name: Ensure docker SDK for python is installed + pip: + name: docker + state: latest + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + - name: Ensure user is in the docker group user: name: "{{ ansible_user_id }}" From ecf0527f97c395fdfc8d9f0a63360610c80daff6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 13 Dec 2017 12:13:47 +0000 Subject: [PATCH 3/7] Use (de)activate-virtualenv role to activate virtualenvs These roles ensure that the previous value of ansible_python_interpreter is restored when the virtualenv is deactivated. --- ansible/roles/ipa-images/tasks/main.yml | 18 ++++++++---------- .../ironic-inspector-rules/tasks/main.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ansible/roles/ipa-images/tasks/main.yml b/ansible/roles/ipa-images/tasks/main.yml index e2b58e24a..1d4aae95a 100644 --- a/ansible/roles/ipa-images/tasks/main.yml +++ b/ansible/roles/ipa-images/tasks/main.yml @@ -29,11 +29,11 @@ - "{{ ipa_images_ramdisk_name }}" register: ipa_images_checksum -# Note that setting this via a play or task variable seems to not -# evaluate the Jinja variable reference, so we use set_fact. -- name: Update the Ansible python interpreter fact to point to the virtualenv - set_fact: - ansible_python_interpreter: "{{ ipa_images_venv }}/bin/python" +- name: Activate the virtualenv + include_role: + name: activate-virtualenv + vars: + activate_virtualenv_path: "{{ ipa_images_venv }}" # To support updating the IPA image, we check the MD5 sum of the cached image # files, and compare with the images in Glance (if there are any). @@ -92,8 +92,6 @@ - name: "{{ ipa_images_ramdisk_name }}" format: ari -# This variable is unset before we set it, and it does not appear to be -# possible to unset a variable in Ansible. -- name: Set a fact to reset the Ansible python interpreter - set_fact: - ansible_python_interpreter: /usr/bin/python +- name: Deactivate the virtualenv + include_role: + name: deactivate-virtualenv diff --git a/ansible/roles/ironic-inspector-rules/tasks/main.yml b/ansible/roles/ironic-inspector-rules/tasks/main.yml index 38cd51bee..2ede1e114 100644 --- a/ansible/roles/ironic-inspector-rules/tasks/main.yml +++ b/ansible/roles/ironic-inspector-rules/tasks/main.yml @@ -8,9 +8,11 @@ with_items: - name: python-ironic-inspector-client -- name: Set a fact to ensure Ansible uses the python interpreter in the virtualenv - set_fact: - ansible_python_interpreter: "{{ ironic_inspector_venv }}/bin/python" +- name: Activate the virtualenv + include_role: + name: activate-virtualenv + vars: + activate_virtualenv_path: "{{ ironic_inspector_venv }}" - name: Ensure introspection rules exist os_ironic_inspector_rule: @@ -24,8 +26,6 @@ inspector_url: "{{ ironic_inspector_url }}" with_items: "{{ ironic_inspector_rules }}" -# This variable is unset before we set it, and it does not appear to be -# possible to unset a variable in Ansible. -- name: Set a fact to reset the Ansible python interpreter - set_fact: - ansible_python_interpreter: /usr/bin/python +- name: Deactivate the virtualenv + include_role: + name: deactivate-virtualenv From 301e7bcb259d2f3f53e7da015b1d2c73496e4505 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 7 Dec 2017 19:02:56 +0000 Subject: [PATCH 4/7] CLI changes and release notes for remote virtualenvs --- doc/source/configuration/kayobe.rst | 16 ++ doc/source/configuration/kolla-ansible.rst | 17 ++ doc/source/release-notes.rst | 14 ++ kayobe/cli/commands.py | 81 +++++++- kayobe/tests/unit/cli/test_commands.py | 219 ++++++++++++++++++++- 5 files changed, 332 insertions(+), 15 deletions(-) diff --git a/doc/source/configuration/kayobe.rst b/doc/source/configuration/kayobe.rst index da974e861..4dd43fd7a 100644 --- a/doc/source/configuration/kayobe.rst +++ b/doc/source/configuration/kayobe.rst @@ -115,3 +115,19 @@ configuration files may be encrypted. Since encryption can make working with Kayobe difficult, it is recommended to follow `best practice `_, adding a layer of indirection and using encryption only where necessary. + +Remote Execution Environment +---------------------------- + +By default, ansible executes modules remotely using the system python +interpreter, even if the ansible control process is executed from within a +virtual environment (unless the ``local`` connection plugin is used). +This is not ideal if there are python dependencies that must be installed +without isolation from the system python packages. Ansible can be configured to +use a virtualenv by setting the host variable ``ansible_python_interpreter`` +to a path to a python interpreter in an existing virtual environment. + +If kayobe detects that ``ansible_python_interpreter`` is set and references a +virtual environment, it will create the virtual environment if it does not +exist. Typically this variable should be set via a group variable for hosts in +the ``seed``, ``seed-hypervisor``, and/or ``overcloud`` groups. diff --git a/doc/source/configuration/kolla-ansible.rst b/doc/source/configuration/kolla-ansible.rst index 261b6b17a..a5d152bd5 100644 --- a/doc/source/configuration/kolla-ansible.rst +++ b/doc/source/configuration/kolla-ansible.rst @@ -27,6 +27,23 @@ kolla-ansible is installed and executed. the kolla-ansible virtualenv will be created. ====================== ================================================== ============================ +Remote Execution Environment +============================ + +By default, ansible executes modules remotely using the system python +interpreter, even if the ansible control process is executed from within a +virtual environment (unless the ``local`` connection plugin is used). +This is not ideal if there are python dependencies that must be installed +without isolation from the system python packages. Ansible can be configured to +use a virtualenv by setting the host variable ``ansible_python_interpreter`` +to a path to a python interpreter in an existing virtual environment. + +If the variable ``kolla_ansible_target_venv`` is set, kolla-ansible will be +configured to create and use a virtual environment on the remote hosts. +This variable is by default set to ``{{ virtualenv_path }}/kolla-ansible``. +The previous behaviour of installing python dependencies directly to the host +can be used by setting ``kolla_ansible_target_venv`` to ``None``. + Control Plane Services ====================== diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index 9e7aa545a..53862be9c 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -31,6 +31,16 @@ Features * Adds commands for management of baremetal compute nodes - ``kayobe baremetal compute inspect``, ``kayobe baremetal compute manage``, and ``kayobe baremetal compute provide``. +* Adds support for installation and use of a python virtual environment for + remote execution of ansible modules, providing isolation from the system's + python packages. This is enabled by setting a host variable, + ``ansible_python_interpreter``, to the path to a python interpreter in a + virtualenv, noting that Jinja2 templating is not supported for this variable. +* Adds support for configuration of a python virtual environment for remote + execution of ansible modules in kolla-ansible, providing isolation from the + system's python packages. This is enabled by setting the variable + ``kolla_ansible_target_venv`` to a path to the virtualenv. The default for + this variable is ``{{ virtualenv_path }}/kolla-ansible``. Upgrade Notes ------------- @@ -56,6 +66,10 @@ Upgrade Notes images for the seed were built on the seed, and container images for the overcloud were built on the controllers. The new design is intended to encourage a build, push, pull workflow. +* The default behaviour is now to configure kolla-ansible to use a virtual + environment for remote execution of ansible modules. The previous behaviour + of installing python dependencies directly to the host can be used by + setting ``kolla_ansible_target_venv`` to ``None`` Kayobe 3.0.0 ============ diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 0d92af956..804ddf41a 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -263,6 +263,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, * Allocate IP addresses for all configured networks. * Add the host to SSH known hosts. + * Optionally, create a virtualenv for remote target hosts. * Configure user accounts, group associations, and authorised SSH keys. * Configure Yum repos. * Configure the host's network interfaces. @@ -274,8 +275,9 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, def take_action(self, parsed_args): self.app.LOG.debug("Configuring seed hypervisor host OS") playbooks = _build_playbook_list( - "ip-allocation", "ssh-known-host", "users", "yum", "dev-tools", - "network", "sysctl", "ntp", "seed-hypervisor-libvirt-host") + "ip-allocation", "ssh-known-host", "kayobe-target-venv", "users", + "yum", "dev-tools", "network", "sysctl", "ntp", + "seed-hypervisor-libvirt-host") self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed-hypervisor") @@ -319,6 +321,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Allocate IP addresses for all configured networks. * Add the host to SSH known hosts. * Configure a user account for use by kayobe for SSH access. + * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. * Configure Yum repos. @@ -329,6 +332,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Disable bootstrap interface configuration. * Configure NTP. * Configure LVM volumes. + * Optionally, create a virtualenv for kolla-ansible. * Configure a user account for kolla-ansible. * Configure Docker engine. """ @@ -344,14 +348,25 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, def take_action(self, parsed_args): self.app.LOG.debug("Configuring seed host OS") - ansible_user = self.run_kayobe_config_dump( - parsed_args, host="seed", var_name="kayobe_ansible_user") + + # Query some kayobe ansible variables. + hostvars = self.run_kayobe_config_dump(parsed_args, hosts="seed") + if not hostvars: + self.app.LOG.error("No hosts in the seed group") + sys.exit(1) + hostvars = hostvars.values()[0] + ansible_user = hostvars.get("kayobe_ansible_user") if not ansible_user: self.app.LOG.error("Could not determine kayobe_ansible_user " "variable for seed host") sys.exit(1) + python_interpreter = hostvars.get("ansible_python_interpreter") + kolla_target_venv = hostvars.get("kolla_ansible_target_venv") + + # Run kayobe playbooks. playbooks = _build_playbook_list( - "ip-allocation", "ssh-known-host", "kayobe-ansible-user") + "ip-allocation", "ssh-known-host", "kayobe-ansible-user", + "kayobe-target-venv") if parsed_args.wipe_disks: playbooks += _build_playbook_list("wipe-disks") playbooks += _build_playbook_list( @@ -360,8 +375,25 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") playbooks = _build_playbook_list("kolla-ansible") self.run_kayobe_playbooks(parsed_args, playbooks, tags="config") + + # Run kolla-ansible bootstrap-servers. + # This command should be run as the kayobe ansible user because at this + # point the kolla user may not exist. + extra_vars = {"ansible_user": ansible_user} + if python_interpreter: + # Use the kayobe virtualenv, as this is the executing user. + extra_vars["ansible_python_interpreter"] = python_interpreter + elif kolla_target_venv: + # Override the kolla-ansible virtualenv, use the system python + # instead. + extra_vars["ansible_python_interpreter"] = "/usr/bin/python" + if kolla_target_venv: + # Specify a virtualenv in which to install python packages. + extra_vars["virtualenv"] = kolla_target_venv self.run_kolla_ansible_seed(parsed_args, "bootstrap-servers", - extra_vars={"ansible_user": ansible_user}) + extra_vars=extra_vars) + + # Run final kayobe playbooks. playbooks = _build_playbook_list("kolla-host", "docker") self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") @@ -559,6 +591,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Allocate IP addresses for all configured networks. * Add the host to SSH known hosts. * Configure a user account for use by kayobe for SSH access. + * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. * Configure Yum repos. @@ -568,6 +601,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Disable bootstrap interface configuration. * Configure NTP. * Configure LVM volumes. + * Optionally, create a virtualenv for kolla-ansible. * Configure a user account for kolla-ansible. * Configure Docker engine. """ @@ -583,15 +617,25 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, def take_action(self, parsed_args): self.app.LOG.debug("Configuring overcloud host OS") - ansible_user = self.run_kayobe_config_dump( - parsed_args, hosts="overcloud", var_name="kayobe_ansible_user") + + # Query some kayobe ansible variables. + hostvars = self.run_kayobe_config_dump(parsed_args, hosts="overcloud") + if not hostvars: + self.app.LOG.error("No hosts in the overcloud group") + sys.exit(1) + hostvars = hostvars.values()[0] + ansible_user = hostvars.get("kayobe_ansible_user") if not ansible_user: self.app.LOG.error("Could not determine kayobe_ansible_user " "variable for overcloud hosts") sys.exit(1) - ansible_user = ansible_user.values()[0] + python_interpreter = hostvars.get("ansible_python_interpreter") + kolla_target_venv = hostvars.get("kolla_ansible_target_venv") + + # Kayobe playbooks. playbooks = _build_playbook_list( - "ip-allocation", "ssh-known-host", "kayobe-ansible-user") + "ip-allocation", "ssh-known-host", "kayobe-ansible-user", + "kayobe-target-venv") if parsed_args.wipe_disks: playbooks += _build_playbook_list("wipe-disks") playbooks += _build_playbook_list( @@ -600,9 +644,26 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") playbooks = _build_playbook_list("kolla-ansible") self.run_kayobe_playbooks(parsed_args, playbooks, tags="config") + + # Kolla-ansible bootstrap-servers. + # The kolla-ansible bootstrap-servers command should be run as the + # kayobe ansible user because at this point the kolla user may not + # exist. extra_vars = {"ansible_user": ansible_user} + if python_interpreter: + # Use the kayobe virtualenv, as this is the executing user. + extra_vars["ansible_python_interpreter"] = python_interpreter + elif kolla_target_venv: + # Override the kolla-ansible virtualenv, use the system python + # instead. + extra_vars["ansible_python_interpreter"] = "/usr/bin/python" + if kolla_target_venv: + # Specify a virtualenv in which to install python packages. + extra_vars["virtualenv"] = kolla_target_venv self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers", extra_vars=extra_vars) + + # Further kayobe playbooks. playbooks = _build_playbook_list("kolla-host", "docker") self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index c1ddd81ee..51fc5db9d 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -98,6 +98,7 @@ class TestCase(unittest.TestCase): [ "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", + "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", "ansible/dev-tools.yml", @@ -121,13 +122,15 @@ class TestCase(unittest.TestCase): command = commands.SeedHostConfigure(TestApp(), []) parser = command.get_parser("test") parsed_args = parser.parse_args([]) - mock_dump.return_value = "stack" + mock_dump.return_value = { + "seed": {"kayobe_ansible_user": "stack"} + } result = command.run(parsed_args) self.assertEqual(0, result) expected_calls = [ - mock.call(mock.ANY, host="seed", var_name="kayobe_ansible_user") + mock.call(mock.ANY, hosts="seed") ] self.assertEqual(expected_calls, mock_dump.call_args_list) @@ -138,6 +141,7 @@ class TestCase(unittest.TestCase): "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", "ansible/kayobe-ansible-user.yml", + "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", "ansible/dev-tools.yml", @@ -177,6 +181,108 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_seed") + def test_seed_host_configure_kayobe_venv(self, mock_kolla_run, mock_run, + mock_dump): + command = commands.SeedHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "seed": { + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "kayobe_ansible_user": "stack", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "ansible_user": "stack", + }, + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_seed") + def test_seed_host_configure_kolla_venv(self, mock_kolla_run, mock_run, + mock_dump): + command = commands.SeedHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "seed": { + "kayobe_ansible_user": "stack", + "kolla_ansible_target_venv": "/kolla/venv/bin/python", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/usr/bin/python", + "ansible_user": "stack", + "virtualenv": "/kolla/venv/bin/python", + }, + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_seed") + def test_seed_host_configure_both_venvs(self, mock_kolla_run, mock_run, + mock_dump): + command = commands.SeedHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "seed": { + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "kayobe_ansible_user": "stack", + "kolla_ansible_target_venv": "/kolla/venv/bin/python", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "ansible_user": "stack", + "virtualenv": "/kolla/venv/bin/python", + }, + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_seed_container_image_build(self, mock_run): @@ -238,15 +344,15 @@ class TestCase(unittest.TestCase): parser = command.get_parser("test") parsed_args = parser.parse_args([]) mock_dump.return_value = { - "controller0": "stack" + "controller0": {"kayobe_ansible_user": "stack"} } result = command.run(parsed_args) self.assertEqual(0, result) expected_calls = [ - mock.call(mock.ANY, hosts="overcloud", - var_name="kayobe_ansible_user")] + mock.call(mock.ANY, hosts="overcloud") + ] self.assertEqual(expected_calls, mock_dump.call_args_list) expected_calls = [ @@ -256,6 +362,7 @@ class TestCase(unittest.TestCase): "ansible/ip-allocation.yml", "ansible/ssh-known-host.yml", "ansible/kayobe-ansible-user.yml", + "ansible/kayobe-target-venv.yml", "ansible/users.yml", "ansible/yum.yml", "ansible/dev-tools.yml", @@ -293,6 +400,108 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_host_configure_kayobe_venv(self, mock_kolla_run, + mock_run, mock_dump): + command = commands.OvercloudHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "controller0": { + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "kayobe_ansible_user": "stack", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "ansible_user": "stack", + } + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_host_configure_kolla_venv(self, mock_kolla_run, + mock_run, mock_dump): + command = commands.OvercloudHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "controller0": { + "kayobe_ansible_user": "stack", + "kolla_ansible_target_venv": "/kolla/venv/bin/python", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/usr/bin/python", + "ansible_user": "stack", + "virtualenv": "/kolla/venv/bin/python", + } + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_config_dump") + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_host_configure_both_venvs(self, mock_kolla_run, + mock_run, mock_dump): + command = commands.OvercloudHostConfigure(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + mock_dump.return_value = { + "controller0": { + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "kayobe_ansible_user": "stack", + "kolla_ansible_target_venv": "/kolla/venv/bin/python", + } + } + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + "bootstrap-servers", + extra_vars={ + "ansible_python_interpreter": "/kayobe/venv/bin/python", + "ansible_user": "stack", + "virtualenv": "/kolla/venv/bin/python", + } + ), + ] + self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_overcloud_container_image_build(self, mock_run): From 5e8212150545fb3e8ff64565fc7c6cc795090d51 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 14 Dec 2017 16:06:09 +0000 Subject: [PATCH 5/7] Don't fail if compute node is powered off in compute-node-discovery.yml Some BMCs aren't idempotent, and cause ipmitool to fail if the requested power state is already set. --- ansible/compute-node-discovery.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/compute-node-discovery.yml b/ansible/compute-node-discovery.yml index 93cf7548f..b163b86ef 100644 --- a/ansible/compute-node-discovery.yml +++ b/ansible/compute-node-discovery.yml @@ -22,6 +22,10 @@ - name: Ensure baremetal compute nodes are powered off command: ipmitool -U {{ ipmi_username }} -P {{ ipmi_password }} -H {{ ipmi_address }} -I lanplus chassis power off delegate_to: "{{ controller_host }}" + failed_when: + - result | failed + # Some BMCs complain if the node is already powered off. + - "'Command not supported in present state' not in result.stderr" vars: # NOTE: Without this, the controller's ansible_host variable will not # be respected when using delegate_to. From 7f04e9ccc6850481ffcec3f8a580e706aa64fb0e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 19 Dec 2017 14:45:14 +0000 Subject: [PATCH 6/7] Create and install virtualenvs on seed/seed-hv/overcloud host upgrade Adds two new commands: kayobe seed hypervisor host upgrade kayobe seed host upgrade These commands can be used prior to performing an upgrade, in addition to the existing command 'kayobe overcloud host upgrade'. These commands will ensure that if kayobe and kolla-ansible remote virtualenvs are in use, they exist and have all dependencies installed. --- ansible/kayobe-target-venv.yml | 2 + ansible/kolla-target-venv.yml | 44 +++++++++++++++++ doc/source/release-notes.rst | 21 ++++++-- doc/source/upgrading.rst | 19 ++++++- kayobe/cli/commands.py | 36 ++++++++++++-- kayobe/tests/unit/cli/test_commands.py | 68 ++++++++++++++++++++++++++ 6 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 ansible/kolla-target-venv.yml diff --git a/ansible/kayobe-target-venv.yml b/ansible/kayobe-target-venv.yml index 262e4ca04..10cf3ffbd 100644 --- a/ansible/kayobe-target-venv.yml +++ b/ansible/kayobe-target-venv.yml @@ -5,6 +5,8 @@ - name: Ensure a virtualenv exists for kayobe hosts: seed:seed-hypervisor:overcloud gather_facts: False + tags: + - kayobe-target-venv tasks: - name: Set a fact about the kayobe target virtualenv set_fact: diff --git a/ansible/kolla-target-venv.yml b/ansible/kolla-target-venv.yml new file mode 100644 index 000000000..bf015f09c --- /dev/null +++ b/ansible/kolla-target-venv.yml @@ -0,0 +1,44 @@ +--- +# Create a virtualenv for ansible modules to use on the remote target systems +# when running kolla-ansible. + +- name: Ensure a virtualenv exists for kolla-ansible + hosts: seed:seed-hypervisor:overcloud + gather_facts: False + tags: + - kolla-ansible + - kolla-target-venv + tasks: + - block: + - name: Ensure the python-virtualenv package is installed + package: + name: python-virtualenv + state: installed + become: True + + - name: Ensure kolla-ansible virtualenv has the latest version of pip installed + pip: + name: pip + state: latest + virtualenv: "{{ kolla_ansible_target_venv }}" + # Site packages are required for using the yum and selinux python + # modules, which are not available via PyPI. + virtualenv_site_packages: True + become: True + + - name: Ensure kolla-ansible virtualenv has docker SDK for python installed + pip: + name: docker + state: latest + virtualenv: "{{ kolla_ansible_target_venv }}" + become: True + + - name: Ensure kolla-ansible virtualenv has correct ownership + file: + path: "{{ kolla_ansible_target_venv }}" + recurse: True + state: directory + owner: kolla + group: kolla + become: True + when: kolla_ansible_target_venv is not none diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index 53862be9c..035a09308 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -66,10 +66,25 @@ Upgrade Notes images for the seed were built on the seed, and container images for the overcloud were built on the controllers. The new design is intended to encourage a build, push, pull workflow. +* It is now possible to configure kayobe to use a virtual environment for + remote execution of ansible modules. If this is required, the following + commands should be run in order to ensure that the virtual environments exist + on the remote hosts:: + + (kayobe) $ kayobe seed hypervisor host upgrade + (kayobe) $ kayobe seed host upgrade + (kayobe) $ kayobe overcloud host upgrade + * The default behaviour is now to configure kolla-ansible to use a virtual - environment for remote execution of ansible modules. The previous behaviour - of installing python dependencies directly to the host can be used by - setting ``kolla_ansible_target_venv`` to ``None`` + environment for remote execution of ansible modules. In order to ensure the + virtual environment exists on the remote hosts, run the following commands:: + + (kayobe) $ kayobe seed hypervisor host upgrade + (kayobe) $ kayobe seed host upgrade + (kayobe) $ kayobe overcloud host upgrade + + The previous behaviour of installing python dependencies directly to the host + can be used by setting ``kolla_ansible_target_venv`` to ``None``. Kayobe 3.0.0 ============ diff --git a/doc/source/upgrading.rst b/doc/source/upgrading.rst index 84ee26f52..81f551c1b 100644 --- a/doc/source/upgrading.rst +++ b/doc/source/upgrading.rst @@ -80,10 +80,27 @@ To upgrade the Ansible control host:: (kayobe) $ kayobe control host upgrade +Upgrading the Seed Hypervisor +============================= + +Currently, upgrading the seed hypervisor services is not supported. It may +however be necessary to upgrade some host services:: + + (kayobe) $ kayobe seed hypervisor host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. + Upgrading the Seed ================== -Currently, upgrading the seed services is not supported. +Currently, upgrading the seed services is not supported. It may however be +necessary to upgrade some host services:: + + (kayobe) $ kayobe seed host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. Upgrading the Overcloud ======================= diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 804ddf41a..6074a4a4d 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -282,6 +282,21 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, limit="seed-hypervisor") +class SeedHypervisorHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command): + """Upgrade the seed hypervisor host services. + + Performs the changes necessary to make the host services suitable for the + configured OpenStack release. + """ + + def take_action(self, parsed_args): + self.app.LOG.debug("Upgrading seed hypervisor host services") + playbooks = _build_playbook_list( + "kayobe-target-venv", "kolla-target-venv") + self.run_kayobe_playbooks(parsed_args, playbooks, + limit="seed-hypervisor") + + class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, Command): """Provision the seed VM. @@ -398,6 +413,21 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") +class SeedHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): + """Upgrade the seed host services. + + Performs the changes necessary to make the host services suitable for the + configured OpenStack release. + """ + + def take_action(self, parsed_args): + self.app.LOG.debug("Upgrading seed host services") + playbooks = _build_playbook_list( + "kayobe-target-venv", "kolla-target-venv") + self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") + + class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, Command): """Deploy the seed services. @@ -668,8 +698,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") -class OvercloudHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, - Command): +class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command): """Upgrade the overcloud host services. Performs the changes necessary to make the host services suitable for the @@ -679,8 +708,9 @@ class OvercloudHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, def take_action(self, parsed_args): self.app.LOG.debug("Upgrading overcloud host services") playbooks = _build_playbook_list( + "kayobe-target-venv", "kolla-target-venv", "overcloud-docker-sdk-upgrade", "overcloud-etc-hosts-fixup") - self.run_kayobe_playbooks(parsed_args, playbooks) + self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") class OvercloudServiceConfigurationGenerate(KayobeAnsibleMixin, diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 51fc5db9d..c311a895a 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -112,6 +112,28 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_seed_hypervisor_host_upgrade(self, mock_run): + command = commands.SeedHypervisorHostUpgrade(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/kayobe-target-venv.yml", + "ansible/kolla-target-venv.yml", + ], + limit="seed-hypervisor", + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_config_dump") @mock.patch.object(commands.KayobeAnsibleMixin, @@ -283,6 +305,28 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_seed_host_upgrade(self, mock_run): + command = commands.SeedHostUpgrade(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/kayobe-target-venv.yml", + "ansible/kolla-target-venv.yml", + ], + limit="seed", + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_seed_container_image_build(self, mock_run): @@ -502,6 +546,30 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_overcloud_host_upgrade(self, mock_run): + command = commands.OvercloudHostUpgrade(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/kayobe-target-venv.yml", + "ansible/kolla-target-venv.yml", + "ansible/overcloud-docker-sdk-upgrade.yml", + "ansible/overcloud-etc-hosts-fixup.yml", + ], + limit="overcloud", + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_overcloud_container_image_build(self, mock_run): From 20ee6d6613d165810fa2cd7124a53cd67d4a3a0d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 19 Dec 2017 15:55:54 +0000 Subject: [PATCH 7/7] Add a comment about the ansible version limit for kolla-ansible --- ansible/roles/kolla-ansible/templates/requirements.txt.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/kolla-ansible/templates/requirements.txt.j2 b/ansible/roles/kolla-ansible/templates/requirements.txt.j2 index 71325f5b9..583728426 100644 --- a/ansible/roles/kolla-ansible/templates/requirements.txt.j2 +++ b/ansible/roles/kolla-ansible/templates/requirements.txt.j2 @@ -5,4 +5,6 @@ # Install Kolla Ansible from PyPI. kolla-ansible=={{ kolla_openstack_release }} {% endif %} +# Limit the version of ansible used by kolla-ansible to avoid new releases from +# breaking tested code. Changes to this limit should be tested. ansible<2.4