From dcac05a30c3da78f93ffb0d37fc1b03a83f1ff4b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 31 Mar 2020 18:58:57 +0100 Subject: [PATCH] Support custom Ansible configuration Tuning Ansible is typically done by customising configuration in ansible.cfg. Currently Kayobe adheres to the standard locations for Ansible configuration [1]. This change allows custom Ansible configuration files stored in the kayobe-config repository to be used for execution of Kayobe and Kolla Ansible playbooks. [1] https://docs.ansible.com/ansible/latest/reference_appendices/config.html#ansible-configuration-settings-locations Change-Id: Iab2021b8e88b5a3a2b0f8583f1246ab2c83670e5 Story: 2007494 Task: 39219 --- doc/source/configuration/kayobe.rst | 10 +++ doc/source/configuration/kolla-ansible.rst | 11 +++ kayobe/ansible.py | 22 +++-- kayobe/cli/commands.py | 9 +- kayobe/kolla_ansible.py | 20 ++++- kayobe/tests/unit/test_ansible.py | 55 +++++++++++++ kayobe/tests/unit/test_kolla_ansible.py | 82 +++++++++++++++++++ ...ustom-ansible-config-d0f1bcdf2607e521.yaml | 9 ++ 8 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/custom-ansible-config-d0f1bcdf2607e521.yaml diff --git a/doc/source/configuration/kayobe.rst b/doc/source/configuration/kayobe.rst index 5e300d3d6..4013e382a 100644 --- a/doc/source/configuration/kayobe.rst +++ b/doc/source/configuration/kayobe.rst @@ -107,6 +107,16 @@ Site Localisation and Customisation Site localisation and customisation is applied using Ansible extra-vars files in ``${KAYOBE_CONFIG_PATH}/*.yml``. +Configuration of Ansible +------------------------ + +Ansible configuration is described in detail in the `Ansible documentation +`__. +In addition to the standard locations, Kayobe supports using an Ansible +configuration file located in the Kayobe configuration at +``${KAYOBE_CONFIG_PATH}/ansible.cfg``. Note that if the ``ANSIBLE_CONFIG`` +environment variable is specified it takes precedence over this file. + Encryption of Secrets --------------------- diff --git a/doc/source/configuration/kolla-ansible.rst b/doc/source/configuration/kolla-ansible.rst index 6ac667c98..edf356a3c 100644 --- a/doc/source/configuration/kolla-ansible.rst +++ b/doc/source/configuration/kolla-ansible.rst @@ -11,6 +11,17 @@ executed from there. Kolla Ansible configuration is stored in ``${KAYOBE_CONFIG_PATH}/kolla.yml``. +Configuration of Ansible +======================== + +Ansible configuration is described in detail in the `Ansible documentation +`__. +In addition to the standard locations, Kayobe supports using an Ansible +configuration file located in the Kayobe configuration at +``${KAYOBE_CONFIG_PATH}/kolla/ansible.cfg`` or +``${KAYOBE_CONFIG_PATH}/ansible.cfg``. Note that if the ``ANSIBLE_CONFIG`` +environment variable is specified it takes precedence over this file. + Kolla Ansible Installation ========================== diff --git a/kayobe/ansible.py b/kayobe/ansible.py index 2cb355ee5..dd9a36d29 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -157,6 +157,21 @@ def build_args(parsed_args, playbooks, return cmd +def _get_environment(parsed_args): + """Return an environment dict for executing an Ansible playbook.""" + env = os.environ.copy() + vault.update_environment(parsed_args, env) + # If the configuration path has been specified via --config-path, ensure + # the environment variable is set, so that it can be referenced by + # playbooks. + env.setdefault(CONFIG_PATH_ENV, parsed_args.config_path) + # If a custom Ansible configuration file exists, use it. + ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg") + if utils.is_readable_file(ansible_cfg_path)["result"]: + env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path) + return env + + def run_playbooks(parsed_args, playbooks, extra_vars=None, limit=None, tags=None, quiet=False, check_output=False, verbose_level=None, check=None, @@ -167,12 +182,7 @@ def run_playbooks(parsed_args, playbooks, extra_vars=extra_vars, limit=limit, tags=tags, verbose_level=verbose_level, check=check, ignore_limit=ignore_limit, list_tasks=list_tasks) - env = os.environ.copy() - vault.update_environment(parsed_args, env) - # If the configuration path has been specified via --config-path, ensure - # the environment variable is set, so that it can be referenced by - # playbooks. - env.setdefault(CONFIG_PATH_ENV, parsed_args.config_path) + env = _get_environment(parsed_args) try: utils.run_command(cmd, check_output=check_output, quiet=quiet, env=env) except subprocess.CalledProcessError as e: diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 751234a7a..0ee471bb9 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -256,7 +256,8 @@ class PlaybookRun(KayobeAnsibleMixin, VaultMixin, Command): self.run_kayobe_playbooks(parsed_args, parsed_args.playbook) -class KollaAnsibleRun(KollaAnsibleMixin, VaultMixin, Command): +class KollaAnsibleRun(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Run a Kolla Ansible command. Allows a single kolla-ansible command to be run. For advanced users only. @@ -1041,7 +1042,8 @@ class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") -class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command): +class OvercloudDatabaseBackup(KollaAnsibleMixin, KayobeAnsibleMixin, + VaultMixin, Command): """Backup the overcloud database.""" def get_parser(self, prog_name): @@ -1061,7 +1063,8 @@ class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command): extra_args=extra_args) -class OvercloudDatabaseRecover(KollaAnsibleMixin, VaultMixin, Command): +class OvercloudDatabaseRecover(KollaAnsibleMixin, KayobeAnsibleMixin, + VaultMixin, Command): """Recover the overcloud database.""" def get_parser(self, prog_name): diff --git a/kayobe/kolla_ansible.py b/kayobe/kolla_ansible.py index f18db5e7a..1c50d1405 100644 --- a/kayobe/kolla_ansible.py +++ b/kayobe/kolla_ansible.py @@ -143,6 +143,23 @@ def build_args(parsed_args, command, inventory_filename, extra_vars=None, return cmd +def _get_environment(parsed_args): + """Return an environment dict for executing Kolla Ansible.""" + env = os.environ.copy() + vault.update_environment(parsed_args, env) + # If a custom Ansible configuration file exists, use it. Allow + # etc/kayobe/kolla/ansible.cfg or etc/kayobe/ansible.cfg. + ansible_cfg_path = os.path.join(parsed_args.config_path, "kolla", + "ansible.cfg") + if utils.is_readable_file(ansible_cfg_path)["result"]: + env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path) + else: + ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg") + if utils.is_readable_file(ansible_cfg_path)["result"]: + env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path) + return env + + def run(parsed_args, command, inventory_filename, extra_vars=None, tags=None, quiet=False, verbose_level=None, extra_args=None, limit=None): @@ -154,8 +171,7 @@ def run(parsed_args, command, inventory_filename, extra_vars=None, verbose_level=verbose_level, extra_args=extra_args, limit=limit) - env = os.environ.copy() - vault.update_environment(parsed_args, env) + env = _get_environment(parsed_args) try: utils.run_command(" ".join(cmd), quiet=quiet, shell=True, env=env) except subprocess.CalledProcessError as e: diff --git a/kayobe/tests/unit/test_ansible.py b/kayobe/tests/unit/test_ansible.py index bb5540ca6..d9cc4290f 100644 --- a/kayobe/tests/unit/test_ansible.py +++ b/kayobe/tests/unit/test_ansible.py @@ -322,6 +322,61 @@ class TestCase(unittest.TestCase): quiet=False, env=expected_env) mock_vars.assert_called_once_with("/etc/kayobe") + @mock.patch.object(utils, "run_command") + @mock.patch.object(utils, "is_readable_file") + @mock.patch.object(ansible, "_get_vars_files") + @mock.patch.object(ansible, "_validate_args") + def test_run_playbooks_ansible_cfg(self, mock_validate, mock_vars, + mock_readable, mock_run): + mock_vars.return_value = [] + mock_readable.return_value = {"result": True} + parser = argparse.ArgumentParser() + ansible.add_args(parser) + vault.add_args(parser) + parsed_args = parser.parse_args([]) + ansible.run_playbooks(parsed_args, ["playbook1.yml"]) + expected_cmd = [ + "ansible-playbook", + "--inventory", "/etc/kayobe/inventory", + "playbook1.yml", + ] + expected_env = { + "ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg", + "KAYOBE_CONFIG_PATH": "/etc/kayobe" + } + mock_run.assert_called_once_with(expected_cmd, check_output=False, + quiet=False, env=expected_env) + mock_vars.assert_called_once_with("/etc/kayobe") + mock_readable.assert_called_once_with("/etc/kayobe/ansible.cfg") + + @mock.patch.object(utils, "run_command") + @mock.patch.object(utils, "is_readable_file") + @mock.patch.object(ansible, "_get_vars_files") + @mock.patch.object(ansible, "_validate_args") + def test_run_playbooks_ansible_cfg_env(self, mock_validate, mock_vars, + mock_readable, mock_run): + mock_vars.return_value = [] + mock_readable.return_value = {"result": True} + os.environ["ANSIBLE_CONFIG"] = "/path/to/ansible.cfg" + parser = argparse.ArgumentParser() + ansible.add_args(parser) + vault.add_args(parser) + parsed_args = parser.parse_args([]) + ansible.run_playbooks(parsed_args, ["playbook1.yml"]) + expected_cmd = [ + "ansible-playbook", + "--inventory", "/etc/kayobe/inventory", + "playbook1.yml", + ] + expected_env = { + "ANSIBLE_CONFIG": "/path/to/ansible.cfg", + "KAYOBE_CONFIG_PATH": "/etc/kayobe" + } + mock_run.assert_called_once_with(expected_cmd, check_output=False, + quiet=False, env=expected_env) + mock_vars.assert_called_once_with("/etc/kayobe") + mock_readable.assert_called_once_with("/etc/kayobe/ansible.cfg") + @mock.patch.object(utils, "run_command") @mock.patch.object(ansible, "_get_vars_files") @mock.patch.object(ansible, "_validate_args") diff --git a/kayobe/tests/unit/test_kolla_ansible.py b/kayobe/tests/unit/test_kolla_ansible.py index ddc717454..1a7486f6e 100644 --- a/kayobe/tests/unit/test_kolla_ansible.py +++ b/kayobe/tests/unit/test_kolla_ansible.py @@ -19,6 +19,7 @@ import unittest import mock +from kayobe import ansible from kayobe import kolla_ansible from kayobe import utils from kayobe import vault @@ -32,6 +33,7 @@ class TestCase(unittest.TestCase): @mock.patch.object(kolla_ansible, "_validate_args") def test_run(self, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) parsed_args = parser.parse_args([]) @@ -49,6 +51,7 @@ class TestCase(unittest.TestCase): @mock.patch.object(kolla_ansible, "_validate_args") def test_run_all_the_args(self, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) args = [ @@ -79,6 +82,7 @@ class TestCase(unittest.TestCase): @mock.patch.object(vault, "_ask_vault_pass") def test_run_all_the_long_args(self, mock_ask, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) mock_ask.return_value = "test-pass" @@ -121,6 +125,7 @@ class TestCase(unittest.TestCase): def test_run_vault_password_file(self, mock_update, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) args = [ @@ -147,6 +152,7 @@ class TestCase(unittest.TestCase): mock_vars.return_value = [] parser = argparse.ArgumentParser() mock_run.return_value = "/path/to/kayobe-vault-password-helper" + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) mock_run.assert_called_once_with( @@ -170,6 +176,7 @@ class TestCase(unittest.TestCase): @mock.patch.object(kolla_ansible, "_validate_args") def test_run_func_args(self, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) args = [ @@ -198,10 +205,85 @@ class TestCase(unittest.TestCase): mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False, env={}) + @mock.patch.object(utils, "run_command") + @mock.patch.object(utils, "is_readable_file") + @mock.patch.object(kolla_ansible, "_validate_args") + def test_run_custom_ansible_cfg(self, mock_validate, mock_readable, + mock_run): + mock_readable.return_value = {"result": True} + parser = argparse.ArgumentParser() + ansible.add_args(parser) + kolla_ansible.add_args(parser) + vault.add_args(parser) + parsed_args = parser.parse_args([]) + kolla_ansible.run(parsed_args, "command", "overcloud") + expected_cmd = [ + ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&", + "kolla-ansible", "command", + "--inventory", "/etc/kolla/inventory/overcloud", + ] + expected_cmd = " ".join(expected_cmd) + expected_env = {"ANSIBLE_CONFIG": "/etc/kayobe/kolla/ansible.cfg"} + mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False, + env=expected_env) + mock_readable.assert_called_once_with("/etc/kayobe/kolla/ansible.cfg") + + @mock.patch.object(utils, "run_command") + @mock.patch.object(utils, "is_readable_file") + @mock.patch.object(kolla_ansible, "_validate_args") + def test_run_custom_ansible_cfg_2(self, mock_validate, mock_readable, + mock_run): + mock_readable.side_effect = [{"result": False}, {"result": True}] + parser = argparse.ArgumentParser() + ansible.add_args(parser) + kolla_ansible.add_args(parser) + vault.add_args(parser) + parsed_args = parser.parse_args([]) + kolla_ansible.run(parsed_args, "command", "overcloud") + expected_cmd = [ + ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&", + "kolla-ansible", "command", + "--inventory", "/etc/kolla/inventory/overcloud", + ] + expected_cmd = " ".join(expected_cmd) + expected_env = {"ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg"} + mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False, + env=expected_env) + expected_calls = [ + mock.call("/etc/kayobe/kolla/ansible.cfg"), + mock.call("/etc/kayobe/ansible.cfg"), + ] + self.assertEqual(mock_readable.call_args_list, expected_calls) + + @mock.patch.object(utils, "run_command") + @mock.patch.object(utils, "is_readable_file") + @mock.patch.object(kolla_ansible, "_validate_args") + def test_run_custom_ansible_cfg_env(self, mock_validate, mock_readable, + mock_run): + mock_readable.return_value = {"result": True} + os.environ["ANSIBLE_CONFIG"] = "/path/to/ansible.cfg" + parser = argparse.ArgumentParser() + ansible.add_args(parser) + kolla_ansible.add_args(parser) + vault.add_args(parser) + parsed_args = parser.parse_args([]) + kolla_ansible.run(parsed_args, "command", "overcloud") + expected_cmd = [ + ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&", + "kolla-ansible", "command", + "--inventory", "/etc/kolla/inventory/overcloud", + ] + expected_cmd = " ".join(expected_cmd) + expected_env = {"ANSIBLE_CONFIG": "/path/to/ansible.cfg"} + mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False, + env=expected_env) + mock_readable.assert_called_once_with("/etc/kayobe/kolla/ansible.cfg") + @mock.patch.object(utils, "run_command") @mock.patch.object(kolla_ansible, "_validate_args") def test_run_failure(self, mock_validate, mock_run): parser = argparse.ArgumentParser() + ansible.add_args(parser) kolla_ansible.add_args(parser) vault.add_args(parser) parsed_args = parser.parse_args([]) diff --git a/releasenotes/notes/custom-ansible-config-d0f1bcdf2607e521.yaml b/releasenotes/notes/custom-ansible-config-d0f1bcdf2607e521.yaml new file mode 100644 index 000000000..400e07dc6 --- /dev/null +++ b/releasenotes/notes/custom-ansible-config-d0f1bcdf2607e521.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Adds support for providing custom Ansible configuration files via Kayobe + configuration. For Kayobe the file should be located at + ``${KAYOBE_CONFIG_PATH}/ansible.cfg``. For Kolla Ansible, it may be located + either at ``${KAYOBE_CONFIG_PATH}/kolla/ansible.cfg`` or + ``${KAYOBE_CONFIG_PATH}/ansible.cfg``. A file specified via the + ``ANSIBLE_CONFIG`` environment variable overrides these.