Use handlers to restart services and move dnsmasq to a unit file

These changes further optimise the lxc_host role so that it's using more
of the built in modules and making better use of handlers.

Moving the dnsmasq process to a unit file gives operators the ability to
restart the dnsmasq process if there's an issue with the service. It
also ensures the service stays running as systemd will take better care
of the service by isolating it within a specific cgroup, ensuring good
reporting and memory management, and providing the ability to recover
from failures in an automated way.

Closes-Bug: #1518485
Change-Id: I42d0caa3b12e70a3601c30051eefc067e81a71bb
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

handlers/main.yml

@@ -27,11 +27,20 @@
 - name: Init reload
   command: "initctl reload-configuration"
 
+- name: Restart dbus
+  service:
+    name: "dbus"
+    state: "reloaded"
+
+- name: Restart machined
+  command: "systemctl restart systemd-machined.service"
+
 - name: Restart irqbalance
   service:
     name: "irqbalance"
     state: "restarted"
     enabled: "yes"
+    daemon_reload: yes
 
 - name: Restart bridge
   shell: "ifdown {{ lxc_net_bridge }} || true"
@@ -66,3 +75,14 @@
   file:
     path: "/tmp/meta.tar.xz"
     state: "absent"
+
+- name: Restart dnsmasq
+  service:
+    name: "lxc-dnsmasq"
+    state: "restarted"
+    enabled: "yes"
+    daemon_reload: yes
+  register: _lxc_dnsmasq_service
+  until: _lxc_dnsmasq_service | success
+  retries: 5
+  delay: 5

tasks/lxc_cache_create.yml

@@ -80,6 +80,9 @@
     content: "{{ cache_time }}"
     dest: "{{ cache_path_fact }}/build_id"
 
-- include: "lxc_container_{{ lxc_container_backing_store }}.yml"
-  when:
-    - lxc_container_backing_store is defined
+- include: "{{ item }}"
+  static: no
+  with_first_found:
+    - files:
+        - "lxc_container_{{ lxc_container_backing_store | default('None') }}.yml"
+      skip: true

tasks/lxc_cache_preparation_systemd_old.yml

@@ -18,7 +18,7 @@
 # ==============================================================
 #  In later versions of SystemD this is automatically done for us
 #  by the machinectl cli on first run.
-- name : Create volume
+- name: Create volume
   shell: |
     if [[ "$(ls -lh /var/lib/machines.raw | awk '{print $5}')" != "{{ lxc_host_machine_volume_size }}.0G" ]]; then
       if [[ ! -f "/var/lib/machines.raw" ]]; then
@@ -58,6 +58,8 @@
     src: var-lib-machines.mount
     dest: /etc/systemd/system/var-lib-machines.mount
   register: mount_unit
+  notify:
+    - Reload systemd units
   when:
     - machines_create | changed
 
@@ -69,6 +71,9 @@
     src: systemd-machined.service.j2
     dest: /etc/systemd/system/systemd-machined.service
   register: machined_unit
+  notify:
+    - Reload systemd units
+    - Restart machined
   when:
     - machines_create | changed
 
@@ -80,20 +85,14 @@
     src: org.freedesktop.machine1.conf
     dest: /etc/dbus-1/system.d/org.freedesktop.machine1.conf
   register: machine1_conf
+  notify:
+    - Reload systemd units
+    - Restart dbus
   when:
     - machines_create | changed
 
-- name: Reload the System daemon
-  command: "systemctl daemon-reload"
-  when: >
-    mount_unit | changed or
-    machined_unit | changed or
-    machine1_conf | changed
-
-- name: Restart dbus
-  command: "systemctl reload dbus.service"
-  when:
-    - machine1_conf | changed
+# Ensure lxc networks are running as they're supposed to
+- meta: flush_handlers
 
 # Ignore the Ansible warning here about using 'mount' via the shell module
 # instead of using the mount Ansible module.
@@ -107,11 +106,6 @@
   tags:
     - skip_ansible_lint
 
-- name: Restart machined
-  command: "systemctl restart systemd-machined.service"
-  when:
-    - machined_unit | changed
-
 # Because of this post and it's related bug(s) this is adding the container
 #  volumes the old way. The new way would simply be calling `machinectl`.
 #  * https://www.mail-archive.com/systemd-devel@lists.freedesktop.org/msg28255.html

tasks/lxc_net.yml

@@ -29,6 +29,13 @@
     - lxc-bridge
     - lxc-interfaces
 
+# NOTE(cloudnull): This task cleans up an old post up script that is no longer
+#                  used. Remove this task in "R".
+- name: Remove old post up script
+  file:
+    path: "/etc/sysconfig/network-scripts/ifup-post-{{ lxc_net_bridge }}"
+    state: "absent"
+
 - name: Drop lxc net bridge routes (SUSE)
   template:
     src: "lxc-net-suse-routes.cfg.j2"
@@ -98,6 +105,19 @@
     - lxc-net
     - lxc-interfaces
 
+- name: Create systemd unit for dnsmasq
+  template:
+    src: lxc-dnsmasq-systemd-init.j2
+    dest: "/etc/systemd/system/lxc-dnsmasq.service"
+    owner: root
+    group: root
+    mode: '0644'
+  when:
+    - lxc_net_nat | bool
+  notify:
+    - Reload systemd units
+    - Restart dnsmasq
+
 # Check that the container bridge exists, if not bring it up
 - name: Check Container Bridge exists
   file:

templates/lxc-dnsmasq-systemd-init.j2

@@ -0,0 +1,41 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=lxc dnsmasq service
+After=syslog.target
+After=network.target
+
+[Service]
+Type=forking
+User=root
+
+{% if lxc_net_manage_iptables | bool %}
+ExecStartPre=/usr/local/bin/lxc-system-manage iptables-create
+{% endif %}
+
+ExecStart=/usr/local/bin/lxc-system-manage dnsmasq-start
+
+ExecStop=-/usr/local/bin/lxc-system-manage dnsmasq-stop
+
+{% if lxc_net_manage_iptables | bool %}
+ExecStopPost=-/usr/local/bin/lxc-system-manage iptables-remove
+{% endif %}
+
+PIDFile=/run/lxc/dnsmasq.pid
+
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec=120
+Restart=on-failure
+RestartSec=2
+
+# This creates a specific slice which all services will operate from
+#  The accounting options give us the ability to see resource usage through
+#  the `systemd-cgtop` command.
+Slice=lxc-dnsmasq.slice
+CPUAccounting=true
+BlockIOAccounting=true
+MemoryAccounting=false
+TasksAccounting=true
+
+[Install]
+WantedBy=multi-user.target

templates/lxc-net-bridge.cfg.j2

@@ -6,17 +6,6 @@ iface {{ lxc_net_bridge }} inet static
     netmask {{ lxc_net_netmask }}
 {% if lxc_net_gateway is not none %}
     gateway {{ lxc_net_gateway }}
-{% endif %}
-{% if lxc_net_nat | bool %}
-    # dnsmasq start and stop
-{% if lxc_net_manage_iptables | bool %}
-    post-up /usr/local/bin/lxc-system-manage iptables-create
-{% endif %}
-    post-up /usr/local/bin/lxc-system-manage dnsmasq-start || true
-{% if lxc_net_manage_iptables | bool %}
-    post-down /usr/local/bin/lxc-system-manage iptables-remove
-{% endif %}
-    post-down /usr/local/bin/lxc-system-manage dnsmasq-stop
 {% endif %}
     bridge_fd 0
     bridge_maxwait 0

templates/lxc-net-suseredhat-postup.cfg.j2

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-if [ "{{ item.interface }}" == "{{ lxc_net_bridge }}" ];then
-  if [ "{{ lxc_net_nat }}" == "True" ];then
-    /usr/local/bin/lxc-system-manage iptables-create
-    /usr/local/bin/lxc-system-manage dnsmasq-start || true
-  fi
-fi

vars/redhat-7.yml

@@ -97,10 +97,6 @@ _lxc_cache_distro_packages:
 lxc_cached_network_interfaces:
   - src: "lxc-net-redhat-bridge.cfg.j2"
     dest: "/etc/sysconfig/network-scripts/ifcfg-{{ lxc_net_bridge }}"
-  - src: "lxc-net-suseredhat-postup.cfg.j2"
-    dest: "/etc/sysconfig/network-scripts/ifup-post-{{ lxc_net_bridge }}"
-    mode: "0755"
-    interface: "${DEVICE}"
   - src: "lxc-net-suseredhat-postdown.cfg.j2"
     dest: "/etc/sysconfig/network-scripts/ifdown-post-{{ lxc_net_bridge }}"
     mode: "0755"

vars/suse-42.yml

@@ -100,10 +100,6 @@ _lxc_cache_distro_packages:
 lxc_cached_network_interfaces:
   - src: "lxc-net-suse-bridge.cfg.j2"
     dest: "/etc/sysconfig/network/ifcfg-{{ lxc_net_bridge }}"
-  - src: "lxc-net-suseredhat-postup.cfg.j2"
-    dest: "/etc/sysconfig/network/scripts/ifup-post-{{ lxc_net_bridge }}"
-    mode: "0755"
-    interface: "${1}"
   - src: "lxc-net-suseredhat-postdown.cfg.j2"
     dest: "/etc/sysconfig/network/scripts/ifdown-post-{{ lxc_net_bridge }}"
     mode: "0755"