libvirt: deploy libvirt on the host

In some cases it may be desirable to run libvirt daemon on the host. For
example, when mixing host and container OS distributions.

This change makes it possible to disable the nova_libvirt container, by
setting kolla_enable_nova_libvirt_container to false.

The stackhpc.libvirt-host role is used in order to install and configure
a libvirt daemon on compute hosts when
kolla_enable_nova_libvirt_container is false.

Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/825357
Depends-On: https://review.opendev.org/c/openstack/kayobe-config-dev/+/829225
Depends-On: https://github.com/stackhpc/ansible-role-libvirt-host/pull/51

Story: 2009858
Task: 44495

Change-Id: I73fef63fb886a9d543d2f4231fb009523495edb3

ansible/compute-libvirt-host.yml

@@ -0,0 +1,56 @@
+---
+- name: Ensure the libvirt daemon is configured
+  hosts: compute
+  tags:
+    - libvirt-host
+  tasks:
+    - name: Ensure Ceph package repository is available
+      package:
+        name: "centos-release-ceph-{{ compute_libvirt_ceph_repo_release }}"
+        state: present
+      when:
+        - compute_libvirt_enabled | bool
+        - ansible_facts.distribution in ['CentOS', 'Rocky']
+        - compute_libvirt_ceph_repo_install | bool
+      become: true
+
+    - name: Include stackhpc.libvirt-host role
+      include_role:
+        name: stackhpc.libvirt-host
+      vars:
+        libvirt_host_libvirtd_conf: "{{ compute_libvirt_conf }}"
+        libvirt_host_qemu_conf: "{{ compute_qemu_conf }}"
+        libvirt_host_tcp_listen: "{{ not compute_libvirt_enable_tls | bool }}"
+        libvirt_host_tcp_listen_address: "{{ internal_net_name | net_ip }}:16509"
+        libvirt_host_tls_listen: "{{ compute_libvirt_enable_tls | bool }}"
+        libvirt_host_tls_listen_address: "{{ internal_net_name | net_ip }}:16514"
+        # TLS server and client certificates.
+        libvirt_host_tls_server_cert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['servercert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_server_key: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['serverkey.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_client_cert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientcert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_client_key: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['clientkey.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        libvirt_host_tls_cacert: >-
+          {{ lookup('file', lookup('first_found', lookup_params | combine({'files': ['cacert.pem']})))
+             if libvirt_host_tls_listen | default(False) | bool else '' }}
+        lookup_params:
+          paths: "{{ libvirt_tls_cert_paths }}"
+          skip: true
+        # Support loading libvirt TLS certificates & keys from per-host and
+        # global locations.
+        libvirt_tls_cert_paths: >-
+          {{ (libvirt_tls_cert_dirs | unique | product([inventory_hostname]) | map('path_join') | list +
+              libvirt_tls_cert_dirs | unique | list) | list }}
+        libvirt_tls_cert_dirs:
+          - "{{ kayobe_env_config_path }}/certificates/libvirt"
+          - "{{ kayobe_config_path }}/certificates/libvirt"
+        libvirt_host_enable_efi_support: true
+      when:
+        - compute_libvirt_enabled | bool

ansible/group_vars/all/compute

@@ -161,3 +161,54 @@ compute_firewalld_default_zone:
 # - permanent: true
 # - state: enabled
 compute_firewalld_rules: []
+
+###############################################################################
+# Compute node host libvirt configuration.
+
+# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova
+# is true and kolla_enable_nova_libvirt_container is false.
+compute_libvirt_enabled: "{{ kolla_enable_nova | bool and not kolla_enable_nova_libvirt_container | bool }}"
+
+# A dict of default configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+compute_libvirt_conf_default:
+  auth_tcp: "none"
+  log_level: "{{ compute_libvirtd_log_level }}"
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+compute_libvirt_conf_extra: {}
+
+# A dict of configuration options to write to /etc/libvirt/libvirtd.conf.
+# Default is a combination of compute_libvirt_conf_default and
+# compute_libvirt_conf_extra.
+compute_libvirt_conf: "{{ compute_libvirt_conf_default | combine(compute_libvirt_conf_extra) }}"
+
+# Numerical log level for libvirtd. Default is 3.
+compute_libvirtd_log_level: 3
+
+# A dict of default configuration options to write to
+# /etc/libvirt/qemu.conf.
+compute_qemu_conf_default:
+  max_files: 32768
+  max_processes: 131072
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/qemu.conf.
+compute_qemu_conf_extra: {}
+
+# A dict of configuration options to write to /etc/libvirt/qemu.conf.
+# Default is a combination of compute_qemu_conf_default and
+# compute_qemu_conf_extra.
+compute_qemu_conf: "{{ compute_qemu_conf_default | combine(compute_qemu_conf_extra) }}"
+
+# Whether to enable a libvirt TLS listener. Default is false.
+compute_libvirt_enable_tls: false
+
+# Whether to install a Ceph package repository on CentOS and Rocky hosts.
+# Default is true.
+compute_libvirt_ceph_repo_install: true
+
+# Ceph package repository release to install on CentOS and Rocky hosts when
+# compute_libvirt_ceph_repo_install is true. Default is 'pacific'.
+compute_libvirt_ceph_repo_release: pacific

ansible/group_vars/all/kolla

@@ -553,6 +553,7 @@ kolla_enable_murano: "no"
 kolla_enable_neutron_mlnx: "no"
 kolla_enable_neutron_provider_networks: "no"
 kolla_enable_neutron_sriov: "no"
+kolla_enable_nova_libvirt_container: "yes"
 kolla_enable_octavia: "no"
 kolla_enable_openvswitch: "{{ kolla_enable_neutron | bool }}"
 kolla_enable_ovn: "no"

ansible/kolla-ansible.yml

@@ -103,6 +103,7 @@
         kolla_inspector_netmask: "{{ inspection_net_name | net_mask }}"
         kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}"
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
+        kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
         kolla_enable_host_ntp: false
         docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
         kolla_globals_paths_extra:

ansible/kolla-openstack.yml

@@ -249,3 +249,5 @@
       kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
       kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}"
       kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config"
+      kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
+      kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt"

ansible/roles/kolla-ansible/defaults/main.yml

@@ -236,6 +236,8 @@ kolla_openstack_logging_debug:
 # controllers.
 kolla_nova_compute_ironic_host:
 
+kolla_libvirt_tls:
+
 ###############################################################################
 # Extra free-form configuraton.
 

ansible/roles/kolla-ansible/templates/kolla/globals.yml

@@ -393,6 +393,10 @@ enable_{{ feature_flag }}: {{ hostvars[inventory_hostname]['kolla_enable_' ~ fea
 # Valid options are [ none, novnc, spice, rdp ]
 #nova_console: "novnc"
 
+{% if kolla_libvirt_tls is not none %}
+libvirt_tls: {{ kolla_libvirt_tls | bool }}
+{% endif %}
+
 #################
 # Hyper-V options
 #################

ansible/roles/kolla-ansible/vars/main.yml

@@ -184,6 +184,7 @@ kolla_feature_flags:
   - nova
   - nova_fake
   - nova_horizon_policy_file
+  - nova_libvirt_container
   - nova_serialconsole_proxy
   - nova_ssh
   - octavia

ansible/roles/kolla-openstack/defaults/main.yml

@@ -447,9 +447,19 @@ kolla_extra_neutron_ml2:
 # Whether to enable Nova.
 kolla_enable_nova:
 
+# Whether to enable Nova libvirt container.
+kolla_enable_nova_libvirt_container:
+
 # Free form extra configuration to append to nova.conf.
 kolla_extra_nova:
 
+# Whether libvirt TLS is enabled.
+kolla_libvirt_tls:
+
+# Directory containing libvirt certificates for nova-compute when running
+# libvirt on the host.
+kolla_nova_libvirt_certificates_src:
+
 ###############################################################################
 # Octavia configuration.
 

ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml

@@ -15,7 +15,7 @@ provisioner:
   inventory:
     group_vars:
       all:
-        kolla_extra_config_path:
+        kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/kolla/config
         kolla_enable_aodh: true
         kolla_extra_aodh: |
           [extra-aodh.conf]
@@ -116,9 +116,12 @@ provisioner:
           [extra-ml2_conf.ini]
           foo=bar
         kolla_enable_nova: true
+        kolla_enable_nova_libvirt_container: false
         kolla_extra_nova: |
           [extra-nova.conf]
           foo=bar
+        kolla_libvirt_tls: true
+        kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/nova-libvirt/certificates
         kolla_enable_octavia: true
         kolla_extra_octavia: |
           [extra-octavia.conf]

ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml

@@ -25,3 +25,23 @@
       with_items:
         - "{{ kolla_inspector_ipa_kernel_path }}"
         - "{{ kolla_inspector_ipa_ramdisk_path }}"
+
+    - name: Ensure nova libvirt certificates directory exists
+      local_action:
+        module: file
+        path: "{{ kolla_nova_libvirt_certificates_src }}"
+        state: directory
+
+    # NOTE(mgoddard): Previously we were creating empty files for the kernel
+    # and ramdisk, but this was found to cause ansible to hang on recent
+    # versions of docker. Using non-empty files seems to resolve the issue.
+    # See https://github.com/ansible/ansible/issues/36725.
+    - name: Ensure nova libvirt certificates exist
+      local_action:
+        module: copy
+        content: fake cert
+        dest: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}"
+      with_items:
+        - "cacert.pem"
+        - "clientcert.pem"
+        - "clientkey.pem"

ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py

@@ -50,6 +50,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
      'murano',
      'neutron',
      'nova',
+     'nova/nova-libvirt',
      'octavia',
      'placement',
      'prometheus',
@@ -100,7 +101,10 @@ def test_service_ini_file(host, path):
 @pytest.mark.parametrize(
     'path',
     ['ironic/ironic-agent.initramfs',
-     'ironic/ironic-agent.kernel'])
+     'ironic/ironic-agent.kernel',
+     'nova/nova-libvirt/cacert.pem',
+     'nova/nova-libvirt/clientcert.pem',
+     'nova/nova-libvirt/clientkey.pem'])
 def test_service_non_ini_file(host, path):
     # TODO(mgoddard): Check config file contents.
     path = os.path.join('/etc/kolla/config', path)

ansible/roles/kolla-openstack/tasks/config.yml

@@ -80,6 +80,7 @@
     recurse: true
   with_items: "{{ kolla_openstack_custom_config }}"
   register: find_src_result
+  delegate_to: localhost
 
 - name: Find previously generated extra configuration files
   find:
@@ -91,7 +92,6 @@
 - name: Ensure extra configuration parent directories are present
   file:
     path: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) | dirname }}"
-    recurse: true
     state: directory
     mode: 0750
   with_subelements:

ansible/roles/kolla-openstack/vars/main.yml

@@ -178,6 +178,27 @@ kolla_openstack_custom_config:
     dest: "{{ kolla_node_custom_config_path }}/nova"
     patterns: "*"
     enabled: "{{ kolla_enable_nova }}"
+  # Nova.
+  - src: "{{ kolla_nova_libvirt_certificates_src }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
+    patterns:
+      - clientcert.pem
+      - clientkey.pem
+      - cacert.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}"
+    untemplated:
+      - clientcert.pem
+      - clientkey.pem
+      - cacert.pem
+  - src: "{{ kolla_nova_libvirt_certificates_src }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
+    patterns:
+      - servercert.pem
+      - serverkey.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool }}"
+    untemplated:
+      - servercert.pem
+      - serverkey.pem
   # Octavia.
   - src: "{{ kolla_extra_config_path }}/octavia"
     dest: "{{ kolla_node_custom_config_path }}/octavia"

dev/functions

@@ -10,12 +10,8 @@ set -o pipefail
 function config_defaults {
     # Set default values for kayobe development configuration.
 
-    # Try to detect if we are running in a vagrant VM.
-    if [[ -e /vagrant ]]; then
-        KAYOBE_SOURCE_PATH_DEFAULT=/vagrant
-    else
-        KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)"
-    fi
+    PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    KAYOBE_SOURCE_PATH_DEFAULT="$(dirname ${PARENT})"
 
     # Path to the kayobe source code repository. Typically this will be the
     # Vagrant shared directory.
@@ -392,18 +388,26 @@ function overcloud_deploy {
 
     control_host_bootstrap
 
-    echo "Configuring the controller host"
-    run_kayobe overcloud host configure
-
-    # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname
-    # resolves to IP address of API interface for RabbitMQ. This seems to be
-    # required since https://review.openstack.org/#/c/584427 was merged.
-    echo "Workaround: upgrading the controller host"
-    run_kayobe overcloud host upgrade
-
     if [[ ${KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES} = 1 ]]; then
         echo "Generate TLS certificates"
-        run_kayobe kolla ansible run certificates --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates
+        run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/kolla-ansible.yml -t config
+        # NOTE(mgoddard): There is a chicken and egg when generating libvirt
+        # TLS certificates using the kolla-ansible certificates command, and
+        # host libvirt. The certificates command needs to be able to gather
+        # facts for all hosts, but since the host configure step hasn't been
+        # run, we don't have SSH or the kolla user configured yet. However, we
+        # can't run host configure without the libvirt TLS certificates.
+        # Workaround: add the host to SSH known hosts and SSH as $USER.
+        run_kayobe playbook run $KAYOBE_SOURCE_PATH/ansible/ssh-known-host.yml -l overcloud
+
+        # Avoid populating the fact cache with this weird setup.
+        export ANSIBLE_CACHE_PLUGIN=memory
+        run_kayobe kolla ansible run certificates \
+          --kolla-extra kolla_certificates_dir=${KAYOBE_CONFIG_PATH}/kolla/certificates \
+          --kolla-extra ansible_user=$USER \
+          --kolla-extra ansible_python_interpreter=/usr/bin/python3
+        unset ANSIBLE_CACHE_PLUGIN
+
         # Add CA cert to trust store.
         ca_cert=${KAYOBE_CONFIG_PATH}/kolla/certificates/ca/root.crt
         if [[ -e /etc/debian_version ]]; then
@@ -417,6 +421,15 @@ function overcloud_deploy {
         fi
     fi
 
+    echo "Configuring the controller host"
+    run_kayobe overcloud host configure
+
+    # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname
+    # resolves to IP address of API interface for RabbitMQ. This seems to be
+    # required since https://review.openstack.org/#/c/584427 was merged.
+    echo "Workaround: upgrading the controller host"
+    run_kayobe overcloud host upgrade
+
     # Note: This must currently be before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.

dev/tenks-deploy-config-compute-libvirt-on-host.yml

@@ -0,0 +1,56 @@
+---
+# This file holds the config given to Tenks when running `tenks-deploy.sh`. It
+# assumes the existence of the bridge `breth1`.
+
+node_types:
+  type0:
+    memory_mb: 1024
+    vcpus: 1
+    volumes:
+      # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
+      # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
+      - capacity: 4GiB
+    physical_networks:
+      - physnet1
+    console_log_enabled: true
+    # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g:
+    # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38
+    cpu_mode:
+
+specs:
+  - type: type0
+    count: 2
+    ironic_config:
+      resource_class: test-rc
+      network_interface: flat
+
+nova_flavors:
+  - resource_class: test-rc
+    node_type: type0
+
+physnet_mappings:
+  physnet1: breth1
+
+deploy_kernel: ipa.kernel
+deploy_ramdisk: ipa.initramfs
+
+default_boot_mode: "bios"
+
+# Use the libvirt daemon deployed by Kayobe. Tenks will install libvirt client
+# packages.
+libvirt_host_install_daemon: false
+
+# Configure AppArmor for the pool on Ubuntu.
+libvirt_host_configure_apparmor: true
+
+# Nested virtualisation is not working well in CI currently. Force the use of
+# QEMU.
+libvirt_vm_engine: "qemu"
+
+# QEMU may not be installed on the host, so set the path and avoid
+# autodetection.
+libvirt_vm_emulator: "{% if ansible_facts.os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}"
+
+# Specify a log path in the kolla_logs Docker volume. It is accessible on the
+# host at the same path.
+libvirt_vm_default_console_log_dir: "/var/log/kolla/tenks"

doc/source/configuration/reference/hosts.rst

@@ -1044,3 +1044,154 @@ Ansible's containers do), but may be necessary when building images.
 Docker's live restore feature can be configured via
 ``docker_daemon_live_restore``, although it is disabled by default due to
 issues observed.
+
+Compute libvirt daemon
+======================
+*tags:*
+  | ``libvirt-host``
+
+.. note::
+
+   This section is about the libvirt daemon on compute nodes, as opposed to the
+   seed hypervisor.
+
+Since Yoga, Kayobe provides support for deploying and configuring a libvirt
+host daemon, as an alternative to the ``nova_libvirt`` container support by
+Kolla Ansible. The host daemon is not used by default, but it is possible to
+enable it by setting ``kolla_enable_nova_libvirt_container`` to ``false`` in
+``$KAYOBE_CONFIG_PATH/kolla.yml``.
+
+Migration of hosts from a containerised libvirt to host libvirt is currently
+not supported.
+
+The following options are available in ``$KAYOBE_CONFIG_PATH/compute.yml`` and
+are relevant only when using the libvirt daemon rather than the
+``nova_libvirt`` container:
+
+``compute_libvirt_enabled``
+    Whether to enable a host libvirt daemon. Default is true if
+    ``kolla_enable_nova`` is ``true`` and
+    ``kolla_enable_nova_libvirt_container`` is ``false``.
+``compute_libvirt_conf_default``
+    A dict of default configuration options to write to
+    ``/etc/libvirt/libvirtd.conf``.
+``compute_libvirt_conf_extra``
+    A dict of additional configuration options to write to
+    ``/etc/libvirt/libvirtd.conf``.
+``compute_libvirt_conf``
+    A dict of configuration options to write to ``/etc/libvirt/libvirtd.conf``.
+    Default is a combination of ``compute_libvirt_conf_default`` and
+    ``compute_libvirt_conf_extra``.
+``compute_libvirtd_log_level``
+    Numerical log level for libvirtd. Default is 3.
+``compute_qemu_conf_default``
+    A dict of default configuration options to write to
+    ``/etc/libvirt/qemu.conf``.
+``compute_qemu_conf_extra``
+    A dict of additional configuration options to write to
+    ``/etc/libvirt/qemu.conf``.
+``compute_qemu_conf``
+    A dict of configuration options to write to ``/etc/libvirt/qemu.conf``.
+    Default is a combination of ``compute_qemu_conf_default`` and
+    ``compute_qemu_conf_extra``.
+``compute_libvirt_enable_tls``
+    Whether to enable a libvirt TLS listener. Default is false.
+``compute_libvirt_ceph_repo_install``
+    Whether to install a Ceph package repository on CentOS and Rocky hosts.
+    Default is ``true``.
+``compute_libvirt_ceph_repo_release``
+    Ceph package repository release to install on CentOS and Rocky hosts when
+    ``compute_libvirt_ceph_repo_install`` is ``true``. Default is ``pacific``.
+
+Example: custom libvirtd.conf
+-----------------------------
+
+To customise the libvirt daemon log output to send level 3 to the journal:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_conf_extra:
+     log_outputs: "3:journald"
+
+Example: custom qemu.conf
+-------------------------
+
+To customise QEMU to avoid adding timestamps to logs:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_qemu_conf_extra:
+     log_timestamp: 0
+
+Example: enabling libvirt TLS listener
+--------------------------------------
+
+To enable the libvirt TLS listener:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_enable_tls: true
+
+When the TLS listener is enabled, it is necessary to provide client, server and
+CA certificates. The following files should be provided:
+
+``cacert.pem``
+    CA certificate used to sign client and server certificates.
+``clientcert.pem``
+    Client certificate.
+``clientkey.pem``
+    Client key.
+``servercert.pem``
+    Server certificate.
+``serverkey.pem``
+    Server key.
+
+It is recommended to encrypt the key files using Ansible Vault.
+
+The following paths are searched for these files:
+
+* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/{{ inventory_hostname }}/``
+* ``$KAYOBE_CONFIG_PATH/certificates/libvirt/``
+
+In this way, certificates may be generated for each host, or shared using
+wildcard certificates.
+
+If using Kayobe environments, certificates in the environment take precedence.
+
+Kayobe makes the CA certificate and client certificate and key available to
+Kolla Ansible, for use by the ``nova_compute`` service.
+
+Example: disabling Ceph repository installation
+-----------------------------------------------
+
+On CentOS and Rocky hosts, a CentOS Storage SIG Ceph repository is installed
+that provides more recent Ceph libraries than those available in CentOS/Rocky
+AppStream.  This may be necessary when using Ceph for Cinder volumes or Nova
+ephemeral block devices. In some cases, such as when using local package
+mirrors, the upstream repository may not be appropriate. The installation of
+the repository may be disabled as follows:
+
+.. code-block:: yaml
+   :caption: ``compute.yml``
+
+   compute_libvirt_ceph_repo_install: false
+
+Example: installing additional packages
+---------------------------------------
+
+In some cases it may be useful to install additional packages on compute hosts
+for use by libvirt. The `stackhpc.libvirt-host
+<https://galaxy.ansible.com/stackhpc/libvirt-host>`__ Ansible role supports
+this via the ``libvirt_host_extra_daemon_packages`` variable. The variable
+should be defined via group variables in the Ansible inventory, to avoid
+applying the change to the seed hypervisor. For example, to install the
+``trousers`` package used for accessing TPM hardware:
+
+.. code-block:: yaml
+   :caption: ``inventory/group_vars/compute/libvirt``
+
+   libvirt_host_extra_daemon_packages:
+     - trousers

doc/source/contributor/automated.rst

@@ -131,11 +131,6 @@ For a control plane with Ironic enabled, a "bare metal" instance can be
 deployed. We can use the `Tenks <https://tenks.readthedocs.io/en/latest/>`__
 project to create fake bare metal nodes.
 
-On Ubuntu, the ``nova_libvirt`` image does not contain the ``qemu-utils``
-package necessary for image operations used by Tenks. Install it as follows::
-
-    sudo docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils'
-
 Clone the tenks repository::
 
     git clone https://opendev.org/openstack/tenks.git

etc/kayobe/compute.yml

@@ -143,6 +143,53 @@
 # - state: enabled
 #compute_firewalld_rules:
 
+###############################################################################
+# Compute node host libvirt configuration.
+
+# Whether to enable a host libvirt daemon. Default is true if kolla_enable_nova
+# is true and kolla_enable_nova_libvirt_container is false.
+#compute_libvirt_enabled:
+
+# A dict of default configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+#compute_libvirt_conf_default:
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/libvirtd.conf.
+#compute_libvirt_conf_extra:
+
+# A dict of configuration options to write to /etc/libvirt/libvirtd.conf.
+# Default is a combination of compute_libvirt_conf_default and
+# compute_libvirt_conf_extra.
+#compute_libvirt_conf:
+
+# Numerical log level for libvirtd. Default is 3.
+#compute_libvirtd_log_level:
+
+# A dict of default configuration options to write to
+# /etc/libvirt/qemu.conf.
+#compute_qemu_conf_default:
+
+# A dict of additional configuration options to write to
+# /etc/libvirt/qemu.conf.
+#compute_qemu_conf_extra:
+
+# A dict of configuration options to write to /etc/libvirt/qemu.conf.
+# Default is a combination of compute_qemu_conf_default and
+# compute_qemu_conf_extra.
+#compute_qemu_conf:
+
+# Whether to enable a libvirt TLS listener. Default is false.
+#compute_libvirt_enable_tls:
+
+# Whether to install a Ceph package repository on CentOS and Rocky hosts.
+# Default is true.
+#compute_libvirt_ceph_repo_install:
+
+# Ceph package repository release to install on CentOS and Rocky hosts when
+# compute_libvirt_ceph_repo_install is true. Default is 'pacific'.
+#compute_libvirt_ceph_repo_release:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -1125,6 +1125,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
     * Optionally, create a virtualenv for kolla-ansible.
     * Configure a user account for kolla-ansible.
     * Configure Docker engine.
+    * Configure libvirt.
     """
 
     def get_parser(self, prog_name):
@@ -1157,7 +1158,8 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kolla_ansible_overcloud(parsed_args, "bootstrap-servers")
 
         # Further kayobe playbooks.
-        playbooks = _build_playbook_list("docker", "swift-block-devices")
+        playbooks = _build_playbook_list(
+            "docker", "swift-block-devices", "compute-libvirt-host")
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
 

kayobe/tests/unit/cli/test_commands.py

@@ -1322,6 +1322,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "docker.yml"),
                     utils.get_data_files_path(
                         "ansible", "swift-block-devices.yml"),
+                    utils.get_data_files_path(
+                        "ansible", "compute-libvirt-host.yml"),
                 ],
                 limit="overcloud",
             ),
@@ -1376,6 +1378,8 @@ class TestCase(unittest.TestCase):
                     utils.get_data_files_path("ansible", "docker.yml"),
                     utils.get_data_files_path(
                         "ansible", "swift-block-devices.yml"),
+                    utils.get_data_files_path(
+                        "ansible", "compute-libvirt-host.yml"),
                 ],
                 limit="overcloud",
             ),

playbooks/kayobe-overcloud-base/globals.yml.j2

@@ -22,4 +22,5 @@ kolla_enable_tls_backend: "yes"
 openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
 kolla_admin_openrc_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
 libvirt_tls: "yes"
+certificates_libvirt_output_dir: "{% raw %}{{ kayobe_env_config_path }}{% endraw %}/certificates/libvirt"
 {% endif %}

playbooks/kayobe-overcloud-base/overrides.yml.j2

@@ -42,6 +42,9 @@ kolla_ironic_default_boot_interface: ipxe
 {% endif %}
 
 {% if tls_enabled %}
+kolla_enable_nova_libvirt_container: false
+compute_libvirt_enable_tls: true
+
 kolla_enable_tls_external: "yes"
 kolla_enable_tls_internal: "yes"
 

playbooks/kayobe-overcloud-base/run.yml

@@ -3,6 +3,8 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
     KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES: "{{ tls_enabled | ternary(1, 0) }}"
+    # TODO(mgoddard): Remove this when libvirt on host is used by default.
+    TENKS_CONFIG_PATH: "dev/tenks-deploy-config-compute{% if tls_enabled %}-libvirt-on-host{% endif %}.yml"
   tasks:
     - name: Ensure overcloud is deployed
       shell:
@@ -18,8 +20,6 @@
         executable: /bin/bash
 
     - name: Perform testing of the virtualized machines
-      # We must do this before tenks-deploy as that will stop the nova_libvirt
-      # container
       shell:
         cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm
         chdir: "{{ kayobe_src_dir }}"

releasenotes/notes/libvirt-on-host-ff83f12923cc1f58.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds support for running a libvirt daemon on the host, rather than in a
+    container. This is done by setting ``kolla_enable_nova_libvirt_container``
+    to ``false``. See `story 2009858
+    <https://storyboard.openstack.org/#!/story/2009858>`__ for details.

requirements.yml

@@ -32,7 +32,7 @@ roles:
   - src: stackhpc.grafana-conf
     version: 1.1.1
   - src: stackhpc.libvirt-host
-    version: v1.8.3
+    version: v1.10.0
   - src: stackhpc.libvirt-vm
     version: v1.14.2
   - src: stackhpc.luks