Adds support for configuring chrony

Some hosts in the kayobe inventory might not be in the kolla-ansible
inventory so it makes sense for kayobe to manage NTP.

Change-Id: Iacb579a46b0e9769a4c404a858d17968f74dd7e0
Depends-On: https://review.opendev.org/c/openstack/kayobe-config-dev/+/786040
Story: 2007872
Task: 40240

ansible/group_vars/all/kolla

@@ -470,7 +470,7 @@ kolla_enable_barbican: "no"
 kolla_enable_blazar: "no"
 kolla_enable_ceilometer: "no"
 kolla_enable_central_logging: "no"
-kolla_enable_chrony: "yes"
+kolla_enable_chrony: "no"
 kolla_enable_cinder: "no"
 kolla_enable_cinder_backend_iscsi: "{{ kolla_enable_cinder_backend_lvm | bool or kolla_enable_cinder_backend_zfssa_iscsi | bool }}"
 kolla_enable_cinder_backend_lvm: "no"

ansible/group_vars/all/time

@@ -6,3 +6,36 @@
 
 # Name of the local timezone.
 timezone: "{{ ansible_date_time.tz }}"
+
+###############################################################################
+# Network Time Protocol (NTP).
+
+# Kayobe default time sources
+chrony_ntp_servers_default:
+  - server: pool.ntp.org
+    type: pool
+    options:
+      - option: iburst
+      - option: minpoll
+        val: 8
+
+# List of NTP time sources to configure. Format is a list of dictionaries with
+# the following keys:
+# server:  host or pool
+# type:    (Optional) Defaults to server. Maps to a time source in the
+#           configuration file. Can be one of server, peer, pool.
+# options: (Optional) List of options that depends on type, see Chrony
+#          documentation for details.
+# See: https://chrony.tuxfamily.org/doc/4.0/chrony.conf.html
+#
+# Example of configuring a pool and customising the pool specific maxsources
+# option:
+# chrony_ntp_servers:
+#   - server: pool.ntp.org
+#     type: pool
+#     options:
+#       - option: maxsources
+#         val: 3
+#
+chrony_ntp_servers: "{{ chrony_ntp_servers_default }}"
+###############################################################################

ansible/roles/ntp/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+ntp_actions: ["validate", "prepare", "deploy"]
+
+ntp_service_disable_list:
+  - ntp.service
+  - systemd-timesyncd.service

ansible/roles/ntp/tasks/deploy.yml

@@ -0,0 +1,4 @@
+---
+- name: Import role to configure chrony
+  import_role:
+    name: mrlesmithjr.chrony

ansible/roles/ntp/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Validate configuration
+  include_tasks: validate.yml
+  when: '"validate" in ntp_actions'
+
+- name: Pre-deploy preparation
+  include_tasks: prepare.yml
+  when: '"prepare" in ntp_actions'
+
+- name: Deploy service
+  include_tasks: deploy.yml
+  when: '"deploy" in ntp_actions'

ansible/roles/ntp/tasks/prepare.yml

@@ -0,0 +1,27 @@
+---
+
+- name: Populate service facts.
+  service_facts:
+
+- name: Mask alternative NTP clients to prevent conflicts
+  vars:
+    service_exists: "{{ item in services }}"
+  systemd:
+    name: "{{ item }}"
+    enabled: "{{ 'false' if service_exists else omit }}"
+    masked: true
+    state: "{{ 'stopped' if service_exists else omit }}"
+  become: true
+  with_items: "{{ ntp_service_disable_list }}"
+
+- name: Remove kolla-ansible installed chrony container
+  docker_container:
+    name: chrony
+    state: absent
+  become: true
+  # NOTE(wszumski):  There is an ordering issue where on a fresh host, docker
+  # will not have been configured, but if that is the case, the chrony container
+  # can't possibly exist, but trying to execute this unconditionally will fail
+  # with: No module named 'docker' as we have not yet added the docker package
+  # to the kayobe target venv.
+  when: "'docker.service' in services"

ansible/roles/ntp/tasks/validate.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Validate structure of chrony_ntp_servers dictionary
+  assert:
+    that:
+      - chrony_ntp_servers is sequence
+      - chrony_ntp_servers | selectattr('server', 'undefined') | list | length == 0
+    msg: "chrony_ntp_servers set to invalid value"
+  when:
+    - chrony_ntp_servers is defined
+    - chrony_ntp_servers | length > 0

ansible/time.yml

@@ -0,0 +1,18 @@
+---
+- name: Ensure timezone is configured
+  hosts: seed-hypervisor:seed:overcloud
+  tags:
+    - timezone
+  tasks:
+    - import_role:
+        name: yatesr.timezone
+      become: True
+
+- name: Ensure Chrony is installed and configured
+  hosts: ntp
+  tags:
+    - ntp
+  tasks:
+    - import_role:
+        name: ntp
+      when: not kolla_enable_chrony | bool

ansible/timezone.yml

@@ -1,8 +1,22 @@
 ---
-- name: Ensure timezone is configured
+# Timezone configuration has moved to time.yml.
-  hosts: seed-hypervisor:seed:overcloud
+# This will be removed in the Xena release.
-  tags:
+
-    - timezone
+# NOTE(wszumski): Making this a non-empty playbook has the benefit of
-  roles:
+# silencing the tox syntax check which doesn't like empty playbooks.
-    - role: yatesr.timezone
+
-      become: True
+- hosts: localhost
+  tasks:
+    - name: Warn about deprecation of this playbook
+      fail:
+        msg: |
+          This playbook has been deprecated, please use time.yml instead.
+          Kayobe should not run this playbook, so if you are seeing this
+          message then either something has gone wrong, or you are trying
+          to run it manually. This playbook will be removed in the Xena
+          release.
+      # NOTE(wszumski): We want this to print a nice big red warning and
+      # not to fail the run.
+      ignore_errors: yes
+
+- import_playbook: time.yml

doc/source/configuration/reference/hosts.rst

@@ -406,23 +406,48 @@ timezone. For example:
 
 NTP
 ===
+*tags:*
+  | ``ntp``
 
-Since the Ussuri release, Kayobe no longer supports configuration of an NTP
+Kayobe will configure `Chrony <https://chrony.tuxfamily.org/>`__ on all hosts in the
-daemon on the host, since the ``ntp`` package is no longer available in CentOS
+``ntp`` group. The default hosts in this group are::
-8.
 
-Kolla Ansible can deploy a chrony container on overcloud hosts, and from the
+.. code-block:: console
-Ussuri release chrony is enabled by default. There is no support for running a
-chrony container on the seed or seed hypervisor hosts.
 
-To disable the containerised chrony daemon, set the following in
+    [ntp:children]
-``${KAYOBE_CONFIG_PATH}/kolla.yml``:
+    # Kayobe will configure Chrony on members of this group.
+    seed
+    seed-hypervisor
+    overcloud
+
+This provides a flexible way to opt in or out of having kayobe manage
+the NTP service.
+
+Variables
+---------
+
+Network Time Protocol (NTP) may be configured via variables in
+``${KAYOBE_CONFIG_PATH}/time.yml``. The list of NTP servers is
+configured via ``chrony_ntp_servers``, and by default the ``pool.ntp.org``
+servers are used.
+
+Internally, kayobe uses the the `mrlesmithjr.chrony
+<https://galaxy.ansible.com/mrlesmithjr/chrony>`__ Ansible role. Rather than
+maintain a mapping between the ``kayobe`` and ``mrlesmithjr.chrony`` worlds, all
+variables are simply passed through. This means you can use all variables that
+the role defines. For example to change ``chrony_maxupdateskew`` and override
+the kayobe defaults for ``chrony_ntp_servers``:
 
 .. code-block:: yaml
+   :caption: ``time.yml``
 
-   kolla_enable_chrony: false
+   chrony_ntp_servers:
-
+     - server: 0.debian.pool.ntp.org
-.. _configuration-hosts-mdadm:
+       options:
+         - option: iburst
+         - option: minpoll
+           val: 8
+   chrony_maxupdateskew: 150.0
 
 Software RAID
 =============

etc/kayobe/inventory/groups

@@ -42,7 +42,7 @@ storage
 compute
 
 ###############################################################################
-# Docker groups.
+# Service groups.
 
 [docker:children]
 # Hosts in this group will have Docker installed.
@@ -59,6 +59,12 @@ compute
 # registries which may become unsynchronized.
 seed
 
+[ntp:children]
+# Kayobe will configure Chrony on members of this group.
+seed
+seed-hypervisor
+overcloud
+
 ###############################################################################
 # Baremetal compute node groups.
 

etc/kayobe/time.yml

@@ -7,6 +7,32 @@
 # Name of the local timezone.
 #timezone:
 
+###############################################################################
+# Network Time Protocol (NTP).
+
+# Kayobe default time sources
+#chrony_ntp_servers_default:
+
+# List of NTP time sources to configure. Format is a list of dictionaries with
+# the following keys:
+# server:  host or pool
+# type:    (Optional) Defaults to server. Maps to a time source in the
+#           configuration file. Can be one of server, peer, pool.
+# options: (Optional) List of options that depends on type, see Chrony
+#          documentation for details.
+# See: https://chrony.tuxfamily.org/doc/4.0/chrony.conf.html
+#
+# Example of configuring a pool and customising the pool specific maxsources
+# option:
+# chrony_ntp_servers:
+#   - server: pool.ntp.org
+#     type: pool
+#     options:
+#       - option: maxsources
+#         val: 3
+#
+#chrony_ntp_servers:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -414,7 +414,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
     * Configure user accounts, group associations, and authorised SSH keys.
     * Configure the host's network interfaces.
     * Set sysctl parameters.
-    * Configure timezone.
+    * Configure timezone and ntp.
     * Optionally, configure software RAID arrays.
     * Optionally, configure encryption.
     * Configure LVM volumes.
@@ -453,7 +453,7 @@ class SeedHypervisorHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
         if parsed_args.wipe_disks:
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
-            "users", "dev-tools", "network", "sysctl", "timezone",
+            "users", "dev-tools", "network", "sysctl", "time",
             "mdadm", "luks", "lvm", "seed-hypervisor-libvirt-host")
         self.run_kayobe_playbooks(parsed_args, playbooks,
                                   limit="seed-hypervisor")
@@ -574,7 +574,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Set sysctl parameters.
     * Configure IP routing and source NAT.
     * Disable bootstrap interface configuration.
-    * Configure timezone.
+    * Configure timezone and ntp.
     * Optionally, configure software RAID arrays.
     * Optionally, configure encryption.
     * Configure LVM volumes.
@@ -608,7 +608,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
             "users", "dev-tools", "disable-selinux", "network",
-            "sysctl", "ip-routing", "snat", "disable-glean", "timezone",
+            "sysctl", "ip-routing", "snat", "disable-glean", "time",
             "mdadm", "luks", "lvm", "docker-devicemapper",
             "kolla-ansible-user", "kolla-pip", "kolla-target-venv")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed")
@@ -948,7 +948,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Configure the host's network interfaces.
     * Set sysctl parameters.
     * Disable bootstrap interface configuration.
-    * Configure timezone.
+    * Configure timezone and ntp.
     * Optionally, configure software RAID arrays.
     * Optionally, configure encryption.
     * Configure LVM volumes.
@@ -981,7 +981,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
             playbooks += _build_playbook_list("wipe-disks")
         playbooks += _build_playbook_list(
             "users", "dev-tools", "disable-selinux", "network",
-            "sysctl", "disable-glean", "disable-cloud-init", "timezone",
+            "sysctl", "disable-glean", "disable-cloud-init", "time",
             "mdadm", "luks", "lvm", "docker-devicemapper",
             "kolla-ansible-user", "kolla-pip", "kolla-target-venv")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")

kayobe/tests/unit/cli/test_commands.py

@@ -325,7 +325,7 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "dev-tools.yml"),
                     utils.get_data_files_path("ansible", "network.yml"),
                     utils.get_data_files_path("ansible", "sysctl.yml"),
-                    utils.get_data_files_path("ansible", "timezone.yml"),
+                    utils.get_data_files_path("ansible", "time.yml"),
                     utils.get_data_files_path("ansible", "mdadm.yml"),
                     utils.get_data_files_path("ansible", "luks.yml"),
                     utils.get_data_files_path("ansible", "lvm.yml"),
@@ -500,7 +500,7 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "ip-routing.yml"),
                     utils.get_data_files_path("ansible", "snat.yml"),
                     utils.get_data_files_path("ansible", "disable-glean.yml"),
-                    utils.get_data_files_path("ansible", "timezone.yml"),
+                    utils.get_data_files_path("ansible", "time.yml"),
                     utils.get_data_files_path("ansible", "mdadm.yml"),
                     utils.get_data_files_path("ansible", "luks.yml"),
                     utils.get_data_files_path("ansible", "lvm.yml"),
@@ -1045,7 +1045,7 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "disable-glean.yml"),
                     utils.get_data_files_path(
                         "ansible", "disable-cloud-init.yml"),
-                    utils.get_data_files_path("ansible", "timezone.yml"),
+                    utils.get_data_files_path("ansible", "time.yml"),
                     utils.get_data_files_path("ansible", "mdadm.yml"),
                     utils.get_data_files_path("ansible", "luks.yml"),
                     utils.get_data_files_path("ansible", "lvm.yml"),

playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2

@@ -122,3 +122,11 @@ dnf_custom_repos:
 # Enable DNF Automatic.
 dnf_automatic_enabled: true
 {% endif %}
+
+# Override the default NTP pool
+chrony_ntp_servers:
+  - server: time.cloudflare.com
+    type: pool
+    options:
+      - option: maxsources
+        val: 2

playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py

@@ -134,6 +134,43 @@ def test_timezone(host):
     assert "Pacific/Honolulu" in status
 
 
+def test_ntp_alternative_services_disabled(host):
+    # Tests that we don't have any conflicting NTP servers running
+    # NOTE(wszumski): We always mask services even if they don't exist
+    ntpd_service = host.service("ntp")
+    assert ntpd_service.is_masked
+    assert not ntpd_service.is_running
+
+    timesyncd_service = host.service("systemd-timesyncd")
+    assert timesyncd_service.is_masked
+    assert not timesyncd_service.is_running
+
+
+def test_ntp_running(host):
+    # Tests that NTP services are enabled and running
+    assert host.package("chrony").is_installed
+    assert host.service("chronyd").is_enabled
+    assert host.service("chronyd").is_running
+
+
+def test_ntp_non_default_time_server(host):
+    # Tests that the NTP pool has been changed from pool.ntp.org to
+    # time.cloudflare.com
+    if 'centos' in host.system_info.distribution.lower():
+        chrony_config = host.file("/etc/chrony.conf")
+    else:
+        # Debian based distributions use the following path
+        chrony_config = host.file("/etc/chrony/chrony.conf")
+    assert chrony_config.exists
+    assert "time.cloudflare.com" in chrony_config.content_string
+
+
+def test_ntp_clock_synchronized(host):
+    # Tests that the clock is synchronized
+    status_output = host.check_output("timedatectl status")
+    assert "synchronized: yes" in status_output
+
+
 @pytest.mark.parametrize('repo', ["AppStream", "BaseOS", "Extras", "epel",
                                   "epel-modular"])
 @pytest.mark.skipif(not _is_dnf(), reason="DNF only supported on CentOS 8")

releasenotes/notes/install-chrony-on-the-host-283fa6fdd9cf9b4c.yaml

@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    Updates the NTP implementation from the chrony container deployed by
+    kolla-ansible to configuring chrony as a host service.  Chrony is now
+    installed on all hosts in the ``ntp`` group, which defaults to include
+    the seed, overcloud, and seed-hypervisor groups. On existing deployments,
+    you should run `kayobe overcloud host configure` to migrate from the
+    kolla-ansible deployed container. This can optionally be scoped to just
+    use the ``ntp`` tag. You can continue to use the kolla container by
+    setting `kolla_enable_chrony` to ``true``.

requirements.yml

@@ -8,6 +8,8 @@
   version: 8438592c84585c86e62ae07e526d3da53629b377
 - src: MichaelRigart.interfaces
   version: v1.11.1
+- src: mrlesmithjr.chrony
+  version: v0.1.0
 - src: mrlesmithjr.manage-lvm
   version: v0.1.4
 - src: mrlesmithjr.mdadm