libvirt: make it possible to run libvirt on the host

In some cases it may be desirable to run the libvirt daemon on the host.
For example, when mixing host and container OS distributions or
versions.

This change makes it possible to disable the nova_libvirt container, by
setting enable_nova_libvirt_container to false. The default values of
some Docker mounts and other paths have been updated to point to default
host directories rather than Docker volumes when using a host libvirt
daemon.

This change does not handle migration of existing systems from using
a nova_libvirt container to libvirt on the host.

Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/830504

Change-Id: Ia1239069ccee39416b20959cbabad962c56693cf

ansible/group_vars/all.yml

@@ -689,6 +689,7 @@ enable_neutron_trunk: "no"
 enable_neutron_metering: "no"
 enable_neutron_infoblox_ipam_agent: "no"
 enable_neutron_port_forwarding: "no"
+enable_nova_libvirt_container: "{{ nova_compute_virt_type in ['kvm', 'qemu'] }}"
 enable_nova_serialconsole_proxy: "no"
 enable_nova_ssh: "yes"
 enable_octavia: "no"
@@ -1013,7 +1014,7 @@ nova_backend_ceph: "no"
 nova_backend: "{{ 'rbd' if nova_backend_ceph | bool else 'default' }}"
 # Valid options are [ kvm, qemu, vmware ]
 nova_compute_virt_type: "kvm"
-nova_instance_datadir_volume: "nova_compute"
+nova_instance_datadir_volume: "{{ 'nova_compute' if enable_nova_libvirt_container | bool else '/var/lib/nova' }}"
 nova_safety_upgrade: "no"
 # Valid options are [ none, novnc, spice ]
 nova_console: "novnc"

ansible/roles/ceilometer/defaults/main.yml

@@ -134,7 +134,7 @@ ceilometer_compute_default_volumes:
   - "/run/:/run/:shared"
   - "ceilometer:/var/lib/ceilometer/"
   - "kolla_logs:/var/log/kolla/"
-  - "nova_libvirt:/var/lib/libvirt"
+  - "{{ ceilometer_libvirt_volume }}:/var/lib/libvirt"
   - "{{ kolla_dev_repos_directory ~ '/ceilometer/ceilometer:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ceilometer' if ceilometer_dev_mode | bool else '' }}"
 ceilometer_ipmi_default_volumes:
   - "{{ node_config_directory }}/ceilometer-ipmi/:{{ container_config_directory }}/:ro"
@@ -150,6 +150,8 @@ ceilometer_central_extra_volumes: "{{ ceilometer_extra_volumes }}"
 ceilometer_compute_extra_volumes: "{{ ceilometer_extra_volumes }}"
 ceilometer_ipmi_extra_volumes: "{{ ceilometer_extra_volumes }}"
 
+ceilometer_libvirt_volume: "{{ 'nova_libvirt' if enable_nova_libvirt_container | bool else '/var/lib/libvirt' }}"
+
 ####################
 # OpenStack
 ####################

ansible/roles/common/tasks/config.yml

@@ -95,7 +95,7 @@
       - name: "conf/input/04-openstack-wsgi.conf.j2"
         enabled: true
       - name: "conf/input/05-libvirt.conf.j2"
-        enabled: true
+        enabled: "{{ enable_nova | bool and enable_nova_libvirt_container | bool }}"
       - name: "conf/input/06-zookeeper.conf.j2"
         enabled: true
       - name: "conf/input/07-kafka.conf.j2"
@@ -186,7 +186,7 @@
       - { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
       - { name: "neutron-tls-proxy", enabled: "{{ neutron_enable_tls_backend | bool }}" }
       - { name: "nova", enabled: "{{ enable_nova | bool }}" }
-      - { name: "nova-libvirt", enabled: "{{ enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}" }
+      - { name: "nova-libvirt", enabled: "{{ enable_nova | bool and enable_nova_libvirt_container | bool }}" }
       - { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
       - { name: "openvswitch", enabled: "{{ enable_openvswitch | bool }}" }
       - { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }

ansible/roles/nova-cell/defaults/main.yml

@@ -3,7 +3,7 @@ nova_cell_services:
   nova-libvirt:
     container_name: nova_libvirt
     group: "{{ nova_cell_compute_group }}"
-    enabled: "{{ nova_compute_virt_type in ['kvm', 'qemu'] }}"
+    enabled: "{{ enable_nova_libvirt_container }}"
     image: "{{ nova_libvirt_image_full }}"
     pid_mode: "host"
     cgroupns_mode: "host"
@@ -399,7 +399,7 @@ nova_compute_default_volumes:
   - "/dev:/dev"
   - "kolla_logs:/var/log/kolla/"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
-  - "libvirtd:/var/lib/libvirt"
+  - "{{ nova_libvirt_volume }}:/var/lib/libvirt"
   - "{{ nova_instance_datadir_volume }}:/var/lib/nova/"
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova/nova:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/nova' if nova_dev_mode | bool else '' }}"
@@ -429,6 +429,8 @@ nova_compute_ironic_extra_volumes: "{{ nova_extra_volumes }}"
 # Used by bootstrapping containers.
 nova_cell_bootstrap_extra_volumes: "{{ nova_extra_volumes }}"
 
+nova_libvirt_volume: "{{ 'libvirtd' if enable_nova_libvirt_container | bool else '/var/lib/libvirt' }}"
+
 ####################
 # HAProxy
 ####################

ansible/roles/nova-cell/handlers/main.yml

@@ -128,6 +128,12 @@
   changed_when: true
   no_log: true
 
+- name: Reload libvirtd
+  become: true
+  service:
+    name: libvirtd
+    state: reloaded
+
 - name: Restart nova-compute container
   vars:
     service_name: "nova-compute"

ansible/roles/nova-cell/tasks/config-host.yml

@@ -39,6 +39,7 @@
     dest: "/etc/udev/rules.d/99-kolla-kvm.rules"
     mode: "0644"
   when:
+    - enable_nova_libvirt_container | bool
     - nova_compute_virt_type == 'kvm'
     - inventory_hostname in groups[nova_cell_compute_group]
 
@@ -51,6 +52,7 @@
     name: qemu-kvm.service
     masked: true
   when:
+    - enable_nova_libvirt_container | bool
     - nova_compute_virt_type == 'kvm'
     - ansible_facts.distribution == 'Ubuntu'
     - inventory_hostname in groups[nova_cell_compute_group]

ansible/roles/nova-cell/tasks/config.yml

@@ -94,6 +94,7 @@
 - name: Copying over libvirt TLS keys
   include_tasks: config-libvirt-tls.yml
   when:
+    - inventory_hostname in groups[nova_cell_compute_group]
     - libvirt_tls | bool
     - libvirt_tls_manage_certs | bool
 

ansible/roles/nova-cell/tasks/external_ceph.yml

@@ -1,14 +1,4 @@
 ---
-- name: Ensuring config directory exists
-  file:
-    path: "{{ node_config_directory }}/{{ item }}"
-    state: "directory"
-    mode: "0770"
-  become: true
-  with_items:
-    - "nova-libvirt/secrets"
-  when: inventory_hostname in groups[nova_cell_compute_group]
-
 - name: Check nova keyring file
   stat:
     path: "{{ node_custom_config }}/nova/{{ ceph_nova_keyring }}"
@@ -31,10 +21,32 @@
     - cinder_backend_ceph | bool
     - external_ceph_cephx_enabled | bool
 
+- name: Extract nova key from file
+  set_fact:
+    nova_cephx_raw_key:
+      "{{ lookup('template', nova_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
+  changed_when: false
+  run_once: True
+  when:
+    - nova_backend == "rbd"
+    - external_ceph_cephx_enabled | bool
+
+- name: Extract cinder key from file
+  set_fact:
+    cinder_cephx_raw_key:
+      "{{ lookup('file', cinder_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
+  changed_when: false
+  run_once: True
+  when:
+    - cinder_backend_ceph | bool
+    - external_ceph_cephx_enabled | bool
+
 - name: Copy over ceph nova keyring file
   template:
     src: "{{ nova_cephx_keyring_file.stat.path }}"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:
@@ -50,6 +62,8 @@
   template:
     src: "{{ cinder_cephx_keyring_file.stat.path }}"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:  # NOTE: nova-libvirt does not need it
@@ -62,90 +76,119 @@
     - Restart {{ item }} container
 
 - name: Copy over ceph.conf
+  vars:
+    service: "{{ nova_cell_services[item] }}"
   template:
     src: "{{ node_custom_config }}/nova/ceph.conf"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:
     - nova-compute
     - nova-libvirt
   when:
-    - inventory_hostname in groups[nova_cell_compute_group]
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
     - nova_backend == "rbd"
   notify:
     - Restart {{ item }} container
 
-- name: Pushing nova secret xml for libvirt
-  template:
-    src: "secret.xml.j2"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml"
-    mode: "0600"
-  become: true
-  when:
-    - inventory_hostname in groups[nova_cell_compute_group]
-    - item.enabled | bool
-  with_items:
-    - uuid: "{{ rbd_secret_uuid }}"
-      name: "client.nova secret"
-      enabled: "{{ nova_backend == 'rbd' }}"
-    - uuid: "{{ cinder_rbd_secret_uuid }}"
-      name: "client.cinder secret"
-      enabled: "{{ cinder_backend_ceph }}"
-  notify:
-    - Restart nova-libvirt container
+- block:
+    - name: Ensure /etc/ceph directory exists (host libvirt)
+      file:
+        path: "/etc/ceph/"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0755"
+      become: true
 
-- name: Extract nova key from file
-  set_fact:
-    nova_cephx_raw_key:
-      "{{ lookup('template', nova_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
-  changed_when: false
-  run_once: True
+    - name: Copy over ceph.conf (host libvirt)
+      template:
+        src: "{{ node_custom_config }}/nova/ceph.conf"
+        dest: "/etc/ceph/ceph.conf"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      become: true
   when:
+    - not enable_nova_libvirt_container | bool
+    - inventory_hostname in groups[nova_cell_compute_group]
     - nova_backend == "rbd"
-    - external_ceph_cephx_enabled | bool
 
-- name: Extract cinder key from file
-  set_fact:
-    cinder_cephx_raw_key:
-      "{{ lookup('template', cinder_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
-  changed_when: false
-  run_once: True
-  when:
-    - cinder_backend_ceph | bool
-    - external_ceph_cephx_enabled | bool
+- block:
+    - name: Ensuring libvirt secrets directory exists
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      file:
+        path: "{{ libvirt_secrets_dir }}"
+        state: "directory"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0770"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
 
-- name: Pushing secrets key for libvirt
-  copy:
-    content: "{{ item.result }}"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64"
-    mode: "0600"
-  become: true
-  when:
-    - inventory_hostname in groups[nova_cell_compute_group]
-    - item.enabled | bool
-    - external_ceph_cephx_enabled | bool
-  with_items:
-    # NOTE(yoctozepto): 'default' filter required due to eager evaluation of item content
-    # which will be undefined if the applicable condition is False
-    - uuid: "{{ rbd_secret_uuid }}"
-      result: "{{ nova_cephx_raw_key | default }}"
-      enabled: "{{ nova_backend == 'rbd' }}"
-    - uuid: "{{ cinder_rbd_secret_uuid }}"
-      result: "{{ cinder_cephx_raw_key | default }}"
-      enabled: "{{ cinder_backend_ceph }}"
-  notify:
-    - Restart nova-libvirt container
-  no_log: True
+    - name: Pushing nova secret xml for libvirt
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      template:
+        src: "secret.xml.j2"
+        dest: "{{ libvirt_secrets_dir }}/{{ item.uuid }}.xml"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0600"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
+        - item.enabled | bool
+      with_items:
+        - uuid: "{{ rbd_secret_uuid }}"
+          name: "client.nova secret"
+          enabled: "{{ nova_backend == 'rbd' }}"
+        - uuid: "{{ cinder_rbd_secret_uuid }}"
+          name: "client.cinder secret"
+          enabled: "{{ cinder_backend_ceph }}"
+      notify: "{{ libvirt_restart_handlers }}"
 
-- name: Ensuring config directory has correct owner and permission
-  become: true
-  file:
-    path: "{{ node_config_directory }}/{{ item }}"
-    recurse: yes
-    owner: "{{ config_owner_user }}"
-    group: "{{ config_owner_group }}"
-  with_items:
-    - "nova-compute"
-    - "nova-libvirt/secrets"
-  when: inventory_hostname in groups[nova_cell_compute_group]
+    - name: Pushing secrets key for libvirt
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      copy:
+        content: "{{ item.result }}"
+        dest: "{{ libvirt_secrets_dir }}/{{ item.uuid }}.base64"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0600"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
+        - item.enabled | bool
+        - external_ceph_cephx_enabled | bool
+      with_items:
+        # NOTE(yoctozepto): 'default' filter required due to eager evaluation of item content
+        # which will be undefined if the applicable condition is False
+        - uuid: "{{ rbd_secret_uuid }}"
+          result: "{{ nova_cephx_raw_key | default }}"
+          enabled: "{{ nova_backend == 'rbd' }}"
+        - uuid: "{{ cinder_rbd_secret_uuid }}"
+          result: "{{ cinder_cephx_raw_key | default }}"
+          enabled: "{{ cinder_backend_ceph }}"
+      notify: "{{ libvirt_restart_handlers }}"
+      no_log: True
+  vars:
+    libvirt_secrets_dir: >-
+      {{ (node_config_directory ~ '/nova-libvirt/secrets')
+         if enable_nova_libvirt_container | bool
+         else '/etc/libvirt/secrets' }}
+    # NOTE(mgoddard): When running libvirt as a host daemon, on CentOS it
+    # appears to pick up secrets automatically, while on Ubuntu it requires a
+    # reload. This may be due to differences in tested versions of libvirt
+    # (8.0.0 vs 6.0.0). Reload should be low overhead, so do it always.
+    libvirt_restart_handlers: >-
+      {{ ['Restart nova-libvirt container']
+         if enable_nova_libvirt_container | bool else
+         ['Reload libvirtd'] }}

ansible/roles/nova-cell/tasks/precheck.yml

@@ -111,13 +111,26 @@
     - nova_libvirt.enabled | bool
     - inventory_hostname in groups[nova_libvirt.group]
 
-- name: Checking that libvirt is not running
+- name: Checking that host libvirt is not running
   vars:
     nova_libvirt: "{{ nova_cell_services['nova-libvirt'] }}"
   stat: path=/var/run/libvirt/libvirt-sock
   register: result
   failed_when: result.stat.exists
   when:
-    - nova_compute_virt_type in ['kvm', 'qemu']
     - container_facts['nova_libvirt'] is not defined
+    - nova_libvirt.enabled | bool
+    - inventory_hostname in groups[nova_libvirt.group]
+
+- name: Checking that nova_libvirt container is not running
+  vars:
+    nova_libvirt: "{{ nova_cell_services['nova-libvirt'] }}"
+  fail:
+    msg: >-
+      The nova_libvirt container is running, however it has been disabled via
+      the 'enable_nova_libvirt_container' flag. Stop and remove the container
+      manually, taking care to migrate any state to the host libvirt daemon.
+  when:
+    - container_facts['nova_libvirt'] is defined
+    - not nova_libvirt.enabled | bool
     - inventory_hostname in groups[nova_libvirt.group]

doc/source/reference/compute/libvirt-guide.rst

@@ -7,7 +7,7 @@ Overview
 
 Libvirt is the most commonly used virtualisation driver in OpenStack. It uses
 libvirt, backed by QEMU and when available, KVM. Libvirt is executed in the
-``nova_libvirt`` container.
+``nova_libvirt`` container, or as a daemon running on the host.
 
 Hardware Virtualisation
 =======================
@@ -42,6 +42,21 @@ The list of enabled authentication mechanisms is configured via
 ``libvirt_sasl_mech_list``, and defaults to ``["SCRAM-SHA-256"]`` if libvirt
 TLS is enabled, or ``["DIGEST-MD5"]`` otherwise.
 
+Host vs containerised libvirt
+=============================
+
+By default, Kolla Ansible deploys libvirt in a ``nova_libvirt`` container. In
+some cases it may be preferable to run libvirt as a daemon on the compute hosts
+instead.
+
+Kolla Ansible does not currently support deploying and configuring
+libvirt as a host daemon. However, since the Yoga release, if a libvirt daemon
+has already been set up, then Kolla Ansible may be configured to use it. This
+may be achieved by setting ``enable_nova_libvirt_container`` to ``false``.
+
+Migration of hosts from a containerised libvirt to host libvirt is currently
+not supported.
+
 .. libvirt-tls:
 
 Libvirt TLS
@@ -77,13 +92,13 @@ will have to supply Kolla Ansible the following pieces of information:
     they can verify that all the certificates being used were signed by the CA
     and should be trusted.
 
-* serverkey.pem
+* serverkey.pem (not used when using a host libvirt daemon)
 
   - This is the private key for the server, and is no different than the
     private key of a TLS certificate. It should be carefully protected, just
     like the private key of a TLS certificate.
 
-* servercert.pem
+* servercert.pem (not used when using a host libvirt daemon)
 
   - This is the public certificate for the server. Libvirt will present this
     certificate to any connection made to the TLS port. This is no different

releasenotes/notes/libvirt-on-host-ce732e057b09fd1f.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds support for running a libvirt daemon on the host, rather than in a
+    container. This is done by setting ``enable_nova_libvirt_container`` to
+    ``false``. Currently this is only supported for fresh deployments without
+    an existing ``nova_libvirt`` container.