Copy custom certificates

Various kolla-ansible TLS features (including backend TLS and custom CA
certs) require certificates to be passed via
$KOLLA_CONFIG_PATH/certificates/. Currently Kayobe does not support
this.

This change adds support for copying across files from
$KAYOBE_CONFIG_PATH/kolla/certificates.

It also uses the kolla-ansible default value for
kolla_external_fqdn_cert and kolla_internal_fqdn_cert when
kolla_external_tls_cert and kolla_internal_tls_cert are respectively
not set. This allows for the standard kolla-ansible configuration
approach of dropping these certificates into the
$KAYOBE_CONFIG_PATH/kolla/certificates directory, rather than defining
them as variables. This can be useful if using the kolla-ansible
certificates command to generate certificates for testing.

Change-Id: I646930ad8ea70991d6ffa00f15f93f72d922141b
Story: 2007679
Task: 39790

ansible/kolla-ansible.yml

@@ -291,6 +291,7 @@
         kolla_internal_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy-internal.pem"
         kolla_ansible_passwords_path: "{{ kayobe_config_path }}/kolla/passwords.yml"
         kolla_overcloud_group_vars_path: "{{ kayobe_config_path }}/kolla/inventory/group_vars"
+        kolla_ansible_certificates_path: "{{ kayobe_config_path }}/kolla/certificates"
         # NOTE: This differs from the default SELinux mode in kolla ansible,
         # which is permissive. The justification for using this mode is twofold:
         # 1. it avoids filling up the audit log

ansible/roles/kolla-ansible/defaults/main.yml

@@ -213,6 +213,7 @@ kolla_neutron_ml2_tenant_network_types: []
 # To provide encryption and authentication on the external and/or internal
 # APIs, TLS can be enabled.  When TLS is enabled, certificates must be provided
 # to allow clients to perform authentication.
+kolla_ansible_certificates_path:
 kolla_enable_tls_external:
 kolla_enable_tls_internal:
 kolla_external_fqdn_cert:

ansible/roles/kolla-ansible/tasks/config.yml

@@ -149,3 +149,45 @@
   when:
     - kolla_internal_tls_cert is not none
     - kolla_internal_tls_cert | length > 0
+
+# Copy across all certificates in $KAYOBE_CONFIG_PATH/kolla/certificates.
+
+- name: Find certificates
+  find:
+    path: "{{ kolla_ansible_certificates_path }}"
+    recurse: true
+  register: find_src_result
+
+- name: Find previously copied certificates
+  find:
+    path: "{{ kolla_config_path }}/certificates"
+    recurse: true
+  register: find_dest_result
+
+- name: Ensure certificates exist
+  copy:
+    src: "{{ kolla_ansible_certificates_path }}/"
+    dest: "{{ kolla_config_path }}/certificates"
+    mode: 0600
+    # If certificates are encrypted, don't decrypt them at the destination.
+    decrypt: false
+  when: find_src_result.files | length > 0
+
+- name: Ensure unnecessary certificates are absent
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ find_dest_result.files }}"
+  when:
+    - item.path | relpath(kolla_config_path ~ '/certificates/') not in src_files
+    - item.path != kolla_external_fqdn_cert
+    - item.path != kolla_internal_fqdn_cert
+  vars:
+    # Find the list of files in the source.
+    src_files: >-
+      {{ find_src_result.files |
+         map(attribute='path') |
+         map('relpath', kolla_ansible_certificates_path) |
+         list }}
+  loop_control:
+    label: "{{ item.path }}"

ansible/roles/kolla-ansible/templates/globals.yml.j2

@@ -176,8 +176,12 @@ neutron_tenant_network_types: {{ kolla_neutron_ml2_tenant_network_types | join('
 # allow clients to perform authentication.
 kolla_enable_tls_internal: {{ kolla_enable_tls_internal | bool }}
 kolla_enable_tls_external: {{ kolla_enable_tls_external | bool }}
+{% if kolla_external_tls_cert is not none and kolla_external_tls_cert | length > 0 %}
 kolla_external_fqdn_cert: "{{ kolla_external_fqdn_cert }}"
+{% endif %}
+{% if kolla_internal_tls_cert is not none and kolla_internal_tls_cert | length > 0 %}
 kolla_internal_fqdn_cert: "{{ kolla_internal_fqdn_cert }}"
+{% endif %}
 kolla_external_fqdn_cacert: "{{ kolla_external_fqdn_cacert }}"
 kolla_internal_fqdn_cacert: "{{ kolla_internal_fqdn_cacert }}"
 

ansible/roles/kolla-ansible/tests/test-defaults.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: localhost
+- name: Test kolla-ansible role defaults
+  hosts: localhost
   connection: local
   tasks:
     - name: Create a temporary directory
@@ -32,10 +33,9 @@
             kolla_internal_fqdn: "fake.internal.fqdn"
             kolla_external_vip_address: "10.0.0.2"
             kolla_external_fqdn: "fake.external.fqdn"
+            kolla_ansible_certificates_path: "{{ temp_path }}/etc/kayobe/kolla/certificates"
             kolla_enable_tls_external: False
-            kolla_external_fqdn_cert: "fake-cert"
             kolla_enable_tls_internal: False
-            kolla_internal_fqdn_cert: "fake-cert"
             kolla_enable_grafana: False
             kolla_openstack_logging_debug: False
 
@@ -74,9 +74,7 @@
               docker_namespace: "fake-namespace"
               neutron_plugin_agent: "openvswitch"
               kolla_enable_tls_external: False
-              kolla_external_fqdn_cert: "fake-cert"
               kolla_enable_tls_internal: False
-              kolla_internal_fqdn_cert: "fake-cert"
               openstack_logging_debug: False
               kolla_user: "kolla"
               kolla_group: "kolla"

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: localhost
+- name: Test kolla-ansible role extras
+  hosts: localhost
   connection: local
   tasks:
     - name: Add a seed host to the inventory
@@ -65,6 +66,23 @@
           ---
           bar_port: "4567"
 
+    - name: Create directory for custom CA certificates
+      file:
+        path: "{{ tempfile_result.path }}/etc/kayobe/kolla/certificates/ca"
+        state: directory
+
+    - name: Create custom CA certificate
+      copy:
+        dest: "{{ tempfile_result.path }}/etc/kayobe/kolla/certificates/ca/foo.crt"
+        content: |
+          bogus CA certificate
+
+    - name: Create custom backend certificate
+      copy:
+        dest: "{{ tempfile_result.path }}/etc/kayobe/kolla/certificates/backend-cert.pem"
+        content: |
+          bogus backend certificate
+
     - block:
         - name: Test the kolla-ansible role with default values
           include_role:
@@ -111,10 +129,15 @@
             kolla_neutron_ml2_tenant_network_types:
               - "fake-ml2-tenant-type-1"
               - "fake-ml2-tenant-type-2"
-            kolla_enable_tls_external: False
+            kolla_ansible_certificates_path: "{{ temp_path }}/etc/kayobe/kolla/certificates"
-            kolla_external_fqdn_cert: "fake-cert"
+            kolla_enable_tls_external: True
-            kolla_enable_tls_internal: False
+            kolla_external_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/external.pem"
-            kolla_internal_fqdn_cert: "fake-cert"
+            kolla_external_tls_cert: |
+              bogus external certificate
+            kolla_enable_tls_internal: True
+            kolla_internal_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/internal.pem"
+            kolla_internal_tls_cert: |
+              bogus internal certificate
             kolla_openstack_logging_debug: True
             grafana_local_admin_user_name: "grafana-admin"
             kolla_inspector_dhcp_pool_start: "1.2.3.4"
@@ -244,10 +267,10 @@
               docker_registry_username: "fake-username"
               docker_registry_password: "fake-password"
               neutron_plugin_agent: "openvswitch"
-              kolla_enable_tls_external: False
+              kolla_enable_tls_external: True
-              kolla_external_fqdn_cert: "fake-cert"
+              kolla_external_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/external.pem"
-              kolla_enable_tls_internal: False
+              kolla_enable_tls_internal: True
-              kolla_internal_fqdn_cert: "fake-cert"
+              kolla_internal_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/internal.pem"
               openstack_logging_debug: True
               grafana_admin_username: "grafana-admin"
               ironic_dnsmasq_dhcp_range: "1.2.3.4,1.2.3.5"
@@ -544,6 +567,84 @@
               - |
                 ---
                 bar_port: "4567"
+
+        - name: Check whether API certificate files exist
+          stat:
+            path: "{{ temp_path ~ '/etc/kolla/certificates/' ~ item }}"
+          with_items:
+            - external.pem
+            - internal.pem
+          register: certificates_stat
+
+        - name: Validate API certificates files
+          assert:
+            that:
+              - item.stat.exists
+              - item.stat.size > 0
+            msg: >
+              API certificate file {{ item.item }} was not found.
+          with_items: "{{ certificates_stat.results }}"
+
+        - name: Read API certificate files
+          slurp:
+            src: "{{ item.stat.path }}"
+          with_items: "{{ certificates_stat.results }}"
+          register: certificates_slurp
+
+        - name: Validate API certificate file contents
+          assert:
+            that:
+              - certificates_content is defined
+              - certificates_content == item.1
+          with_together:
+            - "{{ certificates_slurp.results }}"
+            - "{{ expected_contents }}"
+          vars:
+            certificates_content: "{{ item.0.content | b64decode }}"
+            expected_contents:
+              - |
+                bogus external certificate
+              - |
+                bogus internal certificate
+
+        - name: Check whether custom certificate files exist
+          stat:
+            path: "{{ temp_path ~ '/etc/kolla/certificates/' ~ item }}"
+          with_items:
+            - ca/foo.crt
+            - backend-cert.pem
+          register: certificates_stat
+
+        - name: Validate custom certificates files
+          assert:
+            that:
+              - item.stat.exists
+              - item.stat.size > 0
+            msg: >
+              Custom certificate file {{ item.item }} was not found.
+          with_items: "{{ certificates_stat.results }}"
+
+        - name: Read custom certificate files
+          slurp:
+            src: "{{ item.stat.path }}"
+          with_items: "{{ certificates_stat.results }}"
+          register: certificates_slurp
+
+        - name: Validate custom certificate file contents
+          assert:
+            that:
+              - certificates_content is defined
+              - certificates_content == item.1
+          with_together:
+            - "{{ certificates_slurp.results }}"
+            - "{{ expected_contents }}"
+          vars:
+            certificates_content: "{{ item.0.content | b64decode }}"
+            expected_contents:
+              - |
+                bogus CA certificate
+              - |
+                bogus backend certificate
       always:
         - name: Ensure the temporary directory is removed
           file:

ansible/roles/kolla-ansible/tests/test-requirements.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: localhost
+- name: Test kolla-ansible role requirements
+  hosts: localhost
   connection: local
   tasks:
     - name: Create a temporary directory
@@ -31,10 +32,9 @@
             kolla_internal_fqdn: "fake.internal.fqdn"
             kolla_external_vip_address: "10.0.0.2"
             kolla_external_fqdn: "fake.external.fqdn"
+            kolla_ansible_certificates_path: "{{ temp_path }}/etc/kayobe/kolla/certificates"
             kolla_enable_tls_external: False
             kolla_enable_tls_internal: False
-            kolla_external_fqdn_cert: "fake-cert"
-            kolla_internal_fqdn_cert: "fake-cert"
             kolla_enable_grafana: False
             kolla_openstack_logging_debug: False
 

doc/source/configuration/kolla-ansible.rst

@@ -289,6 +289,57 @@ Here is an example:
      -----END CERTIFICATE-----
    kolla_internal_fqdn_cacert: /path/to/ca/certificate/bundle
 
+Other certificates
+------------------
+
+In general, Kolla Ansible expects certificates to be in a directory configured
+via ``kolla_certificates_dir``, which defaults to a directory named
+``certificates`` in the same directory as ``globals.yml``. Kayobe follows this
+pattern, and will pass files and directories added to
+``${KAYOBE_CONFIG_PATH}/kolla/certificates/`` through to Kolla Ansible. This
+can be useful when enabling backend API TLS encryption, or providing custom CA
+certificates to be added to the trust store in containers. It is also possible
+to use this path to provide certificate bundles for the external or internal
+APIs, as an alternative to ``kolla_external_tls_cert`` and
+``kolla_internal_tls_cert``.
+
+Note that Ansible will automatically decrypt these files if they are encrypted
+via Ansible Vault and it has access to a Vault password.
+
+Example: adding a trusted custom CA certificate to containers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In an environment with a private CA, it may be necessary to add the root CA
+certificate to the trust store of containers.
+
+.. code-block:: console
+   :caption: ``$KAYOBE_CONFIG_PATH``
+
+   kolla/
+     certificates/
+       ca/
+         private-ca.crt
+
+These files should be PEM-formatted, and have a ``.crt`` extension.
+
+Example: adding certificates for backend TLS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Kolla Ansible backend TLS can be used to provide end-to-end encryption of API
+traffic.
+
+.. code-block:: console
+   :caption: ``$KAYOBE_CONFIG_PATH``
+
+   kolla/
+     certificates/
+       backend-cert.pem
+       backend-key.pem
+
+See the :kolla-ansible-doc:`Kolla Ansible documentation
+<admin/advanced-configuration.html#tls-configuration>` for how to provide
+service and/or host-specific certificates and keys.
+
 Custom Global Variables
 -----------------------
 

releasenotes/notes/custom-certificates-5f2c1fff6503b77a.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    Adds support for passing custom TLS certificates to Kolla Ansible via
+    ``${KAYOBE_CONFIG_PATH}/kolla/certificates/``. See `story 2007679
+    <https://storyboard.openstack.org/#!/story/2007679>`__ for details.
+upgrade:
+  - |
+    Reverts to use the Kolla Ansible default value for
+    ``kolla_external_fqdn_cert`` and ``kolla_internal_fqdn_cert`` when
+    ``kolla_external_tls_cert`` and ``kolla_internal_tls_cert`` are
+    respectively not set. This allows for the standard Kolla Ansible
+    configuration approach of dropping these certificates into the
+    ``$KAYOBE_CONFIG_PATH/kolla/certificates`` directory, rather than defining
+    them as variables. This can be useful if using the ``kolla-ansible
+    certificates`` command to generate certificates for testing.