From 01fe797d51189ff5d57366e74004671a7443d5e5 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 21 Feb 2024 11:28:02 +0000 Subject: [PATCH] Support configuring persistent systemd journal This is useful to ensure logs are persisted across reboots. Change-Id: Ie5ff7536b160196dc57448c3ed982d15826b2cbe Closes-Bug: #2055299 Co-Authored-By: Piotr Parczewski --- ansible/infra-vm-host-configure.yml | 1 + ansible/inventory/group_vars/all/logging | 9 ++++ ansible/logging.yml | 51 +++++++++++++++++++ ansible/overcloud-host-configure.yml | 1 + ansible/seed-host-configure.yml | 1 + ansible/seed-hypervisor-host-configure.yml | 1 + doc/source/configuration/reference/hosts.rst | 22 ++++++++ etc/kayobe/logging.yml | 11 ++++ ...s-persistent-journal-99951ac651dc4a23.yaml | 7 +++ 9 files changed, 104 insertions(+) create mode 100644 ansible/inventory/group_vars/all/logging create mode 100644 ansible/logging.yml create mode 100644 etc/kayobe/logging.yml create mode 100644 releasenotes/notes/adds-persistent-journal-99951ac651dc4a23.yaml diff --git a/ansible/infra-vm-host-configure.yml b/ansible/infra-vm-host-configure.yml index 3a987d24c..b1068965a 100644 --- a/ansible/infra-vm-host-configure.yml +++ b/ansible/infra-vm-host-configure.yml @@ -1,6 +1,7 @@ --- - import_playbook: "ssh-known-host.yml" - import_playbook: "kayobe-ansible-user.yml" +- import_playbook: "logging.yml" - import_playbook: "proxy.yml" - import_playbook: "apt.yml" - import_playbook: "dnf.yml" diff --git a/ansible/inventory/group_vars/all/logging b/ansible/inventory/group_vars/all/logging new file mode 100644 index 000000000..a140da97a --- /dev/null +++ b/ansible/inventory/group_vars/all/logging @@ -0,0 +1,9 @@ +--- +############################################################################### +# Logging configuration + +# Journald storage. One of: volatile, persistent, auto, or none. Defaults to +# `persistent`. +journald_storage: persistent + +############################################################################### diff --git a/ansible/logging.yml b/ansible/logging.yml new file mode 100644 index 000000000..ddc584af8 --- /dev/null +++ b/ansible/logging.yml @@ -0,0 +1,51 @@ +--- +- name: Ensure Logging configuration is applied + hosts: seed-hypervisor:seed:overcloud:infra-vms + gather_facts: false + vars: + # NOTE(wszumski): Kayobe target env does not yet exist. + ansible_python_interpreter: /usr/bin/python3 + tags: + - logging + tasks: + - name: Create /var/log/journal + become: true + file: + path: /var/log/journal + owner: "root" + group: "root" + mode: 0755 + state: directory + notify: Flush journal to disk + when: journald_storage == "persistent" + + - name: Ensure journald drop in directory exists + file: + path: "/etc/systemd/journald.conf.d/" + owner: "root" + group: "root" + mode: 0770 + state: directory + become: true + + - name: Ensure journald.conf overrides are set + copy: + content: | + [Journal] + Storage={{ journald_storage }} + dest: /etc/systemd/journald.conf.d/kayobe.conf + owner: root + group: root + mode: 0660 + become: true + notify: Restart journald + + handlers: + - name: Restart journald + service: + name: systemd-journald + state: restarted + become: true + - name: Flush journal to disk + become: true + command: journalctl --flush diff --git a/ansible/overcloud-host-configure.yml b/ansible/overcloud-host-configure.yml index afdfa6b9a..ab2694b01 100644 --- a/ansible/overcloud-host-configure.yml +++ b/ansible/overcloud-host-configure.yml @@ -1,6 +1,7 @@ --- - import_playbook: "ssh-known-host.yml" - import_playbook: "kayobe-ansible-user.yml" +- import_playbook: "logging.yml" - import_playbook: "proxy.yml" - import_playbook: "apt.yml" - import_playbook: "dnf.yml" diff --git a/ansible/seed-host-configure.yml b/ansible/seed-host-configure.yml index 6040eda42..846aa85a4 100644 --- a/ansible/seed-host-configure.yml +++ b/ansible/seed-host-configure.yml @@ -1,6 +1,7 @@ --- - import_playbook: "ssh-known-host.yml" - import_playbook: "kayobe-ansible-user.yml" +- import_playbook: "logging.yml" - import_playbook: "proxy.yml" - import_playbook: "apt.yml" - import_playbook: "dnf.yml" diff --git a/ansible/seed-hypervisor-host-configure.yml b/ansible/seed-hypervisor-host-configure.yml index dbec1a8b5..9bcb1f50c 100644 --- a/ansible/seed-hypervisor-host-configure.yml +++ b/ansible/seed-hypervisor-host-configure.yml @@ -1,6 +1,7 @@ --- - import_playbook: "ssh-known-host.yml" - import_playbook: "kayobe-ansible-user.yml" +- import_playbook: "logging.yml" - import_playbook: "proxy.yml" - import_playbook: "apt.yml" - import_playbook: "dnf.yml" diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 4dd7075ee..4eade8470 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -1393,3 +1393,25 @@ Installing packages required by Kolla Ansible A small number of packages are required to be installed on the hosts for Kolla Ansible and the services that it deploys, while some others must be removed. + +Logging +======= +*tags:* + | ``logging`` + +Kayobe will configure persistent logging for nodes in the following ansible groups: + +- seed-hypervisor +- seed +- overcloud +- infra-vms + +This means that the systemd journal will be written to local storage (instead +of to memory) and will allow you to view the journal from previous boots. The +storage limit defaults to 10% of the filesystem with a 4GiB hard limit (when +using journald defaults). See `journald documentation +`__ +for more details. + +Should you wish to disable this feature, you can set ``journald_storage`` to +``volatile``. \ No newline at end of file diff --git a/etc/kayobe/logging.yml b/etc/kayobe/logging.yml new file mode 100644 index 000000000..46033d268 --- /dev/null +++ b/etc/kayobe/logging.yml @@ -0,0 +1,11 @@ +--- +############################################################################### +# Logging configuration + +# Journald storage. One of: volatile, persistent, auto, or none. Defaults to +# `persistent`. +#journald_storage: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/releasenotes/notes/adds-persistent-journal-99951ac651dc4a23.yaml b/releasenotes/notes/adds-persistent-journal-99951ac651dc4a23.yaml new file mode 100644 index 000000000..eed2ad0d0 --- /dev/null +++ b/releasenotes/notes/adds-persistent-journal-99951ac651dc4a23.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Configures journald to use a persistent storage by default. This allows you + to keep journald logs across reboots and is controlled by the + ``journald_storage`` variable. See :kayobe-doc:`Kayobe documentation + ` for more details.