Support custom Ansible configuration

Tuning Ansible is typically done by customising configuration in
ansible.cfg. Currently Kayobe adheres to the standard locations for
Ansible configuration [1].

This change allows custom Ansible configuration files stored in the
kayobe-config repository to be used for execution of Kayobe and Kolla
Ansible playbooks.

[1] https://docs.ansible.com/ansible/latest/reference_appendices/config.html#ansible-configuration-settings-locations

Change-Id: Iab2021b8e88b5a3a2b0f8583f1246ab2c83670e5
Story: 2007494
Task: 39219

doc/source/configuration/kayobe.rst

@@ -107,6 +107,16 @@ Site Localisation and Customisation
 Site localisation and customisation is applied using Ansible extra-vars files
 in ``${KAYOBE_CONFIG_PATH}/*.yml``.
 
+Configuration of Ansible
+------------------------
+
+Ansible configuration is described in detail in the `Ansible documentation
+<https://docs.ansible.com/ansible/latest/reference_appendices/config.html>`__.
+In addition to the standard locations, Kayobe supports using an Ansible
+configuration file located in the Kayobe configuration at
+``${KAYOBE_CONFIG_PATH}/ansible.cfg``. Note that if the ``ANSIBLE_CONFIG``
+environment variable is specified it takes precedence over this file.
+
 Encryption of Secrets
 ---------------------
 

doc/source/configuration/kolla-ansible.rst

@@ -11,6 +11,17 @@ executed from there.
 
 Kolla Ansible configuration is stored in ``${KAYOBE_CONFIG_PATH}/kolla.yml``.
 
+Configuration of Ansible
+========================
+
+Ansible configuration is described in detail in the `Ansible documentation
+<https://docs.ansible.com/ansible/latest/reference_appendices/config.html>`__.
+In addition to the standard locations, Kayobe supports using an Ansible
+configuration file located in the Kayobe configuration at
+``${KAYOBE_CONFIG_PATH}/kolla/ansible.cfg`` or
+``${KAYOBE_CONFIG_PATH}/ansible.cfg``. Note that if the ``ANSIBLE_CONFIG``
+environment variable is specified it takes precedence over this file.
+
 Kolla Ansible Installation
 ==========================
 

kayobe/ansible.py

@@ -157,6 +157,21 @@ def build_args(parsed_args, playbooks,
     return cmd
 
 
+def _get_environment(parsed_args):
+    """Return an environment dict for executing an Ansible playbook."""
+    env = os.environ.copy()
+    vault.update_environment(parsed_args, env)
+    # If the configuration path has been specified via --config-path, ensure
+    # the environment variable is set, so that it can be referenced by
+    # playbooks.
+    env.setdefault(CONFIG_PATH_ENV, parsed_args.config_path)
+    # If a custom Ansible configuration file exists, use it.
+    ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg")
+    if utils.is_readable_file(ansible_cfg_path)["result"]:
+        env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path)
+    return env
+
+
 def run_playbooks(parsed_args, playbooks,
                   extra_vars=None, limit=None, tags=None, quiet=False,
                   check_output=False, verbose_level=None, check=None,
@@ -167,12 +182,7 @@ def run_playbooks(parsed_args, playbooks,
                      extra_vars=extra_vars, limit=limit, tags=tags,
                      verbose_level=verbose_level, check=check,
                      ignore_limit=ignore_limit, list_tasks=list_tasks)
-    env = os.environ.copy()
+    env = _get_environment(parsed_args)
-    vault.update_environment(parsed_args, env)
-    # If the configuration path has been specified via --config-path, ensure
-    # the environment variable is set, so that it can be referenced by
-    # playbooks.
-    env.setdefault(CONFIG_PATH_ENV, parsed_args.config_path)
     try:
         utils.run_command(cmd, check_output=check_output, quiet=quiet, env=env)
     except subprocess.CalledProcessError as e:

kayobe/cli/commands.py

@@ -256,7 +256,8 @@ class PlaybookRun(KayobeAnsibleMixin, VaultMixin, Command):
         self.run_kayobe_playbooks(parsed_args, parsed_args.playbook)
 
 
-class KollaAnsibleRun(KollaAnsibleMixin, VaultMixin, Command):
+class KollaAnsibleRun(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+                      Command):
     """Run a Kolla Ansible command.
 
     Allows a single kolla-ansible command to be run. For advanced users only.
@@ -1041,7 +1042,8 @@ class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
 
-class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command):
+class OvercloudDatabaseBackup(KollaAnsibleMixin, KayobeAnsibleMixin,
+                              VaultMixin, Command):
     """Backup the overcloud database."""
 
     def get_parser(self, prog_name):
@@ -1061,7 +1063,8 @@ class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command):
                                          extra_args=extra_args)
 
 
-class OvercloudDatabaseRecover(KollaAnsibleMixin, VaultMixin, Command):
+class OvercloudDatabaseRecover(KollaAnsibleMixin, KayobeAnsibleMixin,
+                               VaultMixin, Command):
     """Recover the overcloud database."""
 
     def get_parser(self, prog_name):

kayobe/kolla_ansible.py

@@ -143,6 +143,23 @@ def build_args(parsed_args, command, inventory_filename, extra_vars=None,
     return cmd
 
 
+def _get_environment(parsed_args):
+    """Return an environment dict for executing Kolla Ansible."""
+    env = os.environ.copy()
+    vault.update_environment(parsed_args, env)
+    # If a custom Ansible configuration file exists, use it. Allow
+    # etc/kayobe/kolla/ansible.cfg or etc/kayobe/ansible.cfg.
+    ansible_cfg_path = os.path.join(parsed_args.config_path, "kolla",
+                                    "ansible.cfg")
+    if utils.is_readable_file(ansible_cfg_path)["result"]:
+        env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path)
+    else:
+        ansible_cfg_path = os.path.join(parsed_args.config_path, "ansible.cfg")
+        if utils.is_readable_file(ansible_cfg_path)["result"]:
+            env.setdefault("ANSIBLE_CONFIG", ansible_cfg_path)
+    return env
+
+
 def run(parsed_args, command, inventory_filename, extra_vars=None,
         tags=None, quiet=False, verbose_level=None, extra_args=None,
         limit=None):
@@ -154,8 +171,7 @@ def run(parsed_args, command, inventory_filename, extra_vars=None,
                      verbose_level=verbose_level,
                      extra_args=extra_args,
                      limit=limit)
-    env = os.environ.copy()
+    env = _get_environment(parsed_args)
-    vault.update_environment(parsed_args, env)
     try:
         utils.run_command(" ".join(cmd), quiet=quiet, shell=True, env=env)
     except subprocess.CalledProcessError as e:

kayobe/tests/unit/test_ansible.py

@@ -322,6 +322,61 @@ class TestCase(unittest.TestCase):
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with("/etc/kayobe")
 
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(ansible, "_get_vars_files")
+    @mock.patch.object(ansible, "_validate_args")
+    def test_run_playbooks_ansible_cfg(self, mock_validate, mock_vars,
+                                       mock_readable, mock_run):
+        mock_vars.return_value = []
+        mock_readable.return_value = {"result": True}
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        vault.add_args(parser)
+        parsed_args = parser.parse_args([])
+        ansible.run_playbooks(parsed_args, ["playbook1.yml"])
+        expected_cmd = [
+            "ansible-playbook",
+            "--inventory", "/etc/kayobe/inventory",
+            "playbook1.yml",
+        ]
+        expected_env = {
+            "ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg",
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+        }
+        mock_run.assert_called_once_with(expected_cmd, check_output=False,
+                                         quiet=False, env=expected_env)
+        mock_vars.assert_called_once_with("/etc/kayobe")
+        mock_readable.assert_called_once_with("/etc/kayobe/ansible.cfg")
+
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(ansible, "_get_vars_files")
+    @mock.patch.object(ansible, "_validate_args")
+    def test_run_playbooks_ansible_cfg_env(self, mock_validate, mock_vars,
+                                           mock_readable, mock_run):
+        mock_vars.return_value = []
+        mock_readable.return_value = {"result": True}
+        os.environ["ANSIBLE_CONFIG"] = "/path/to/ansible.cfg"
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        vault.add_args(parser)
+        parsed_args = parser.parse_args([])
+        ansible.run_playbooks(parsed_args, ["playbook1.yml"])
+        expected_cmd = [
+            "ansible-playbook",
+            "--inventory", "/etc/kayobe/inventory",
+            "playbook1.yml",
+        ]
+        expected_env = {
+            "ANSIBLE_CONFIG": "/path/to/ansible.cfg",
+            "KAYOBE_CONFIG_PATH": "/etc/kayobe"
+        }
+        mock_run.assert_called_once_with(expected_cmd, check_output=False,
+                                         quiet=False, env=expected_env)
+        mock_vars.assert_called_once_with("/etc/kayobe")
+        mock_readable.assert_called_once_with("/etc/kayobe/ansible.cfg")
+
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(ansible, "_get_vars_files")
     @mock.patch.object(ansible, "_validate_args")

kayobe/tests/unit/test_kolla_ansible.py

@@ -19,6 +19,7 @@ import unittest
 
 import mock
 
+from kayobe import ansible
 from kayobe import kolla_ansible
 from kayobe import utils
 from kayobe import vault
@@ -32,6 +33,7 @@ class TestCase(unittest.TestCase):
     @mock.patch.object(kolla_ansible, "_validate_args")
     def test_run(self, mock_validate, mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         parsed_args = parser.parse_args([])
@@ -49,6 +51,7 @@ class TestCase(unittest.TestCase):
     @mock.patch.object(kolla_ansible, "_validate_args")
     def test_run_all_the_args(self, mock_validate, mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         args = [
@@ -79,6 +82,7 @@ class TestCase(unittest.TestCase):
     @mock.patch.object(vault, "_ask_vault_pass")
     def test_run_all_the_long_args(self, mock_ask, mock_validate, mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         mock_ask.return_value = "test-pass"
@@ -121,6 +125,7 @@ class TestCase(unittest.TestCase):
     def test_run_vault_password_file(self, mock_update, mock_validate,
                                      mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         args = [
@@ -147,6 +152,7 @@ class TestCase(unittest.TestCase):
         mock_vars.return_value = []
         parser = argparse.ArgumentParser()
         mock_run.return_value = "/path/to/kayobe-vault-password-helper"
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         mock_run.assert_called_once_with(
@@ -170,6 +176,7 @@ class TestCase(unittest.TestCase):
     @mock.patch.object(kolla_ansible, "_validate_args")
     def test_run_func_args(self, mock_validate, mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         args = [
@@ -198,10 +205,85 @@ class TestCase(unittest.TestCase):
         mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False,
                                          env={})
 
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(kolla_ansible, "_validate_args")
+    def test_run_custom_ansible_cfg(self, mock_validate, mock_readable,
+                                    mock_run):
+        mock_readable.return_value = {"result": True}
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        kolla_ansible.add_args(parser)
+        vault.add_args(parser)
+        parsed_args = parser.parse_args([])
+        kolla_ansible.run(parsed_args, "command", "overcloud")
+        expected_cmd = [
+            ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&",
+            "kolla-ansible", "command",
+            "--inventory", "/etc/kolla/inventory/overcloud",
+        ]
+        expected_cmd = " ".join(expected_cmd)
+        expected_env = {"ANSIBLE_CONFIG": "/etc/kayobe/kolla/ansible.cfg"}
+        mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False,
+                                         env=expected_env)
+        mock_readable.assert_called_once_with("/etc/kayobe/kolla/ansible.cfg")
+
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(kolla_ansible, "_validate_args")
+    def test_run_custom_ansible_cfg_2(self, mock_validate, mock_readable,
+                                      mock_run):
+        mock_readable.side_effect = [{"result": False}, {"result": True}]
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        kolla_ansible.add_args(parser)
+        vault.add_args(parser)
+        parsed_args = parser.parse_args([])
+        kolla_ansible.run(parsed_args, "command", "overcloud")
+        expected_cmd = [
+            ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&",
+            "kolla-ansible", "command",
+            "--inventory", "/etc/kolla/inventory/overcloud",
+        ]
+        expected_cmd = " ".join(expected_cmd)
+        expected_env = {"ANSIBLE_CONFIG": "/etc/kayobe/ansible.cfg"}
+        mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False,
+                                         env=expected_env)
+        expected_calls = [
+            mock.call("/etc/kayobe/kolla/ansible.cfg"),
+            mock.call("/etc/kayobe/ansible.cfg"),
+        ]
+        self.assertEqual(mock_readable.call_args_list, expected_calls)
+
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(kolla_ansible, "_validate_args")
+    def test_run_custom_ansible_cfg_env(self, mock_validate, mock_readable,
+                                        mock_run):
+        mock_readable.return_value = {"result": True}
+        os.environ["ANSIBLE_CONFIG"] = "/path/to/ansible.cfg"
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        kolla_ansible.add_args(parser)
+        vault.add_args(parser)
+        parsed_args = parser.parse_args([])
+        kolla_ansible.run(parsed_args, "command", "overcloud")
+        expected_cmd = [
+            ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&",
+            "kolla-ansible", "command",
+            "--inventory", "/etc/kolla/inventory/overcloud",
+        ]
+        expected_cmd = " ".join(expected_cmd)
+        expected_env = {"ANSIBLE_CONFIG": "/path/to/ansible.cfg"}
+        mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False,
+                                         env=expected_env)
+        mock_readable.assert_called_once_with("/etc/kayobe/kolla/ansible.cfg")
+
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(kolla_ansible, "_validate_args")
     def test_run_failure(self, mock_validate, mock_run):
         parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
         kolla_ansible.add_args(parser)
         vault.add_args(parser)
         parsed_args = parser.parse_args([])

releasenotes/notes/custom-ansible-config-d0f1bcdf2607e521.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Adds support for providing custom Ansible configuration files via Kayobe
+    configuration. For Kayobe the file should be located at
+    ``${KAYOBE_CONFIG_PATH}/ansible.cfg``. For Kolla Ansible, it may be located
+    either at ``${KAYOBE_CONFIG_PATH}/kolla/ansible.cfg`` or
+    ``${KAYOBE_CONFIG_PATH}/ansible.cfg``. A file specified via the
+    ``ANSIBLE_CONFIG`` environment variable overrides these.