octavia: generate certificates automatically

implemented as a separate command (kolla-ansible octavia-certificates)

Implements: blueprint implement-automatic-deploy-of-octavia

Co-Authored-By: wu.chunyang <wuchunyang@yovole.com>
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>

Change-Id: I2c5b26ce9e363f35c523865904a582f7960aa682

ansible/octavia-certificates.yml

@@ -0,0 +1,5 @@
+---
+- name: Apply role octavia-certificates
+  hosts: localhost
+  roles:
+    - octavia-certificates

ansible/roles/octavia-certificates/defaults/main.yml

@@ -0,0 +1,45 @@
+---
+#####################
+# Certificate options.
+#####################
+
+octavia_certs_work_dir: "{{ node_config }}/octavia-certificates"
+
+# OpenSSL configuration file path.
+octavia_certs_openssl_cnf_path: openssl.cnf
+
+# For more info see: https://en.wikipedia.org/wiki/Certificate_signing_request
+# Country; The two-letter ISO code for the country where your organization is located
+octavia_certs_country: US
+# Province, Region, County or State
+octavia_certs_state: Oregon
+# Business name / Organization
+octavia_certs_organization: OpenStack
+# Department Name / Organizational Unit
+octavia_certs_organizational_unit: Octavia
+
+# Server CA.
+octavia_certs_server_ca_expiry: 3650
+octavia_certs_server_ca_country: "{{ octavia_certs_country }}"
+octavia_certs_server_ca_state: "{{ octavia_certs_state }}"
+octavia_certs_server_ca_organization: "{{ octavia_certs_organization }}"
+octavia_certs_server_ca_organizational_unit: "{{ octavia_certs_organizational_unit }}"
+octavia_certs_server_ca_common_name: server-ca.example.org
+
+# Client CA.
+octavia_certs_client_ca_expiry: 3650
+octavia_certs_client_ca_country: "{{ octavia_certs_country }}"
+octavia_certs_client_ca_state: "{{ octavia_certs_state }}"
+octavia_certs_client_ca_organization: "{{ octavia_certs_organization }}"
+octavia_certs_client_ca_organizational_unit: "{{ octavia_certs_organizational_unit }}"
+octavia_certs_client_ca_common_name: client-ca.example.org
+
+# Client certificate.
+octavia_certs_client_expiry: 365
+octavia_certs_client_req_country: "{{ octavia_certs_country }}"
+octavia_certs_client_req_state: "{{ octavia_certs_state }}"
+octavia_certs_client_req_organization: "{{ octavia_certs_organization }}"
+octavia_certs_client_req_organizational_unit: "{{ octavia_certs_organizational_unit }}"
+# NOTE(yoctozepto): This should ideally be per controller, i.e. controller
+# generates its key&CSR and this CA signs it.
+octavia_certs_client_req_common_name: client.example.org

ansible/roles/octavia-certificates/files/openssl.cnf

@@ -0,0 +1,49 @@
+[ client_ca ]
+new_certs_dir     = .
+database          = index.txt
+serial            = serial
+RANDFILE          = .rand
+
+private_key       = client_ca.key.pem
+certificate       = client_ca.cert.pem
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md        = sha256
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = 3650
+
+x509_extensions   = client_cert
+
+policy            = policy_any
+
+[ policy_any ]
+countryName            = supplied
+stateOrProvinceName    = optional
+organizationName       = optional
+organizationalUnitName = optional
+commonName             = supplied
+emailAddress           = optional
+
+[ req ]
+distinguished_name = req_distinguished_name
+x509_extensions    = v3_ca
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md         = sha256
+
+[ req_distinguished_name ]
+
+[ v3_ca ]
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always
+basicConstraints = critical, CA:TRUE
+keyUsage = critical, cRLSign, keyCertSign
+
+[ client_cert ]
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always
+basicConstraints = critical, CA:FALSE
+keyUsage = critical, digitalSignature
+extendedKeyUsage = clientAuth

ansible/roles/octavia-certificates/tasks/client_ca.yml

@@ -0,0 +1,43 @@
+---
+
+- name: Create client_ca index.txt
+  copy:
+    content: ''
+    dest: "{{ octavia_certs_work_dir }}/client_ca/index.txt"
+    force: no
+    mode: 0660
+
+- name: Create client_ca serial
+  copy:
+    content: "1000\n"
+    dest: "{{ octavia_certs_work_dir }}/client_ca/serial"
+    force: no
+    mode: 0660
+
+- name: Create client_ca private key
+  command: >
+    openssl genrsa -aes256 -out client_ca.key.pem
+    -passout pass:{{ octavia_client_ca_password }} 4096
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/client_ca"
+    creates: "{{ octavia_certs_work_dir }}/client_ca/client_ca.key.pem"
+
+- name: Create client_ca certificate
+  vars:
+    client_ca_subject:
+      C: "{{ octavia_certs_client_ca_country }}"
+      ST: "{{ octavia_certs_client_ca_state }}"
+      O: "{{ octavia_certs_client_ca_organization }}"
+      OU: "{{ octavia_certs_client_ca_organizational_unit }}"
+      CN: "{{ octavia_certs_client_ca_common_name }}"
+  command: >
+    openssl req -new -x509 -config ../openssl.cnf
+    -key client_ca.key.pem
+    -days {{ octavia_certs_client_ca_expiry }}
+    -out client_ca.cert.pem
+    -subj "/{{ client_ca_subject.items() | map('join', '=') | join('/') }}"
+    -passin pass:{{ octavia_client_ca_password }}
+    -batch
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/client_ca"
+    creates: "{{ octavia_certs_work_dir }}/client_ca/client_ca.cert.pem"

ansible/roles/octavia-certificates/tasks/client_cert.yml

@@ -0,0 +1,50 @@
+---
+
+# NOTE(yoctozepto): This should ideally be per controller, i.e. controller
+# generates its key&CSR and this CA signs it.
+
+- name: Create a key for the client certificate
+  command: >
+    openssl genrsa -out client.key.pem 4096
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/client_ca"
+    creates: "{{ octavia_certs_work_dir }}/client_ca/client.key.pem"
+
+- name: Create the certificate request for the client certificate
+  vars:
+    client_req_subject:
+      C: "{{ octavia_certs_client_req_country }}"
+      ST: "{{ octavia_certs_client_req_state }}"
+      O: "{{ octavia_certs_client_req_organization }}"
+      OU: "{{ octavia_certs_client_req_organizational_unit }}"
+      CN: "{{ octavia_certs_client_req_common_name }}"
+  command: >
+    openssl req -new -config ../openssl.cnf
+    -key client.key.pem
+    -out client.csr.pem
+    -subj "/{{ client_req_subject.items() | map('join', '=') | join('/') }}"
+    -batch
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/client_ca"
+    creates: "{{ octavia_certs_work_dir }}/client_ca/client.csr.pem"
+
+- name: Sign the client certificate request
+  command: >
+    openssl ca -config ../openssl.cnf
+    -name client_ca
+    -days {{ octavia_certs_client_expiry }}
+    -in client.csr.pem
+    -out client.cert.pem
+    -key {{ octavia_client_ca_password }}
+    -notext
+    -batch
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/client_ca"
+    creates: "{{ octavia_certs_work_dir }}/client_ca/client.cert.pem"
+
+- name: Create a concatenated client certificate and key file
+  assemble:
+    regexp: ^client\.(cert|key)\.pem$
+    src: "{{ octavia_certs_work_dir }}/client_ca"
+    dest: "{{ octavia_certs_work_dir }}/client_ca/client.cert-and-key.pem"
+    mode: "0660"

ansible/roles/octavia-certificates/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+# This play adapts https://docs.openstack.org/octavia/victoria/admin/guides/certificates.html
+
+# Kolla-Ansible prepares the Server CA certificate and key for use by Octavia
+# to generate Amphorae certificates.
+
+# Kolla-Ansible prepares and controls the Client CA certificate and key.
+# Client CA is used to generate certificates for Octavia controllers.
+
+- name: Ensure server_ca and client_ca directories exist
+  file:
+    path: "{{ octavia_certs_work_dir }}/{{ item }}"
+    state: "directory"
+    mode: 0770
+  loop:
+    - server_ca
+    - client_ca
+
+- name: Copy openssl.cnf
+  copy:
+    src: "{{ octavia_certs_openssl_cnf_path }}"
+    dest: "{{ octavia_certs_work_dir }}/openssl.cnf"
+
+- import_tasks: server_ca.yml
+
+- import_tasks: client_ca.yml
+
+- import_tasks: client_cert.yml
+
+- name: Ensure {{ node_custom_config }}/octavia directory exists
+  file:
+    path: "{{ node_custom_config }}/octavia"
+    state: "directory"
+    mode: 0770
+
+- name: Copy the to-be-deployed keys and certs to {{ node_custom_config }}/octavia
+  copy:
+    src: "{{ octavia_certs_work_dir }}/{{ item.src }}"
+    dest: "{{ node_custom_config }}/octavia/{{ item.dest }}"
+  with_items:
+    - { src: "server_ca/server_ca.cert.pem", dest: "server_ca.cert.pem" }
+    - { src: "server_ca/server_ca.key.pem", dest: "server_ca.key.pem" }
+    - { src: "client_ca/client_ca.cert.pem", dest: "client_ca.cert.pem" }
+    - { src: "client_ca/client.cert-and-key.pem", dest: "client.cert-and-key.pem" }

ansible/roles/octavia-certificates/tasks/server_ca.yml

@@ -0,0 +1,29 @@
+---
+
+- name: Generate server_ca private key
+  command: >
+    openssl genrsa -aes256 -out server_ca.key.pem
+    -passout pass:{{ octavia_ca_password }} 4096
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/server_ca"
+    creates: "{{ octavia_certs_work_dir }}/server_ca/server_ca.key.pem"
+
+- name: Create server_ca certificate
+  vars:
+    server_ca_subject:
+      C: "{{ octavia_certs_server_ca_country }}"
+      ST: "{{ octavia_certs_server_ca_state }}"
+      O: "{{ octavia_certs_server_ca_organization }}"
+      OU: "{{ octavia_certs_server_ca_organizational_unit }}"
+      CN: "{{ octavia_certs_server_ca_common_name }}"
+  command: >
+    openssl req -new -x509 -config ../openssl.cnf
+    -key server_ca.key.pem
+    -days {{ octavia_certs_server_ca_expiry }}
+    -out server_ca.cert.pem
+    -subj "/{{ server_ca_subject.items() | map('join', '=') | join('/') }}"
+    -passin pass:{{ octavia_ca_password }}
+    -batch
+  args:
+    chdir: "{{ octavia_certs_work_dir }}/server_ca"
+    creates: "{{ octavia_certs_work_dir }}/server_ca/server_ca.cert.pem"

etc/kolla/passwords.yml

@@ -170,6 +170,7 @@ manila_keystone_password:
 octavia_database_password:
 octavia_keystone_password:
 octavia_ca_password:
+octavia_client_ca_password:
 
 searchlight_keystone_password:
 

tests/check-config.sh

@@ -16,6 +16,7 @@ function check_config {
     for f in $(sudo find /etc/kolla \
                 -not -regex /etc/kolla/config.* \
                 -not -regex /etc/kolla/certificates.* \
+                -not -regex /etc/kolla/octavia-certificates.* \
                 -not -regex .*pem \
                 -not -regex .*key \
                 -not -regex ".*ca-certificates.*" \

tests/run.yml

@@ -194,24 +194,6 @@
               dest: ironic-agent.kernel
       when: scenario == "ironic"
 
-    - block:
-        - name: ensure octavia config directory exists
-          file:
-            path: /etc/kolla/config/octavia
-            state: directory
-            mode: 0777
-
-        - name: create dummy TLS certificates for octavia
-          file:
-            path: "/etc/kolla/config/octavia/{{ item }}"
-            state: touch
-          with_items:
-            - client.cert-and-key.pem
-            - client_ca.cert.pem
-            - server_ca.cert.pem
-            - server_ca.key.pem
-      when: scenario == 'magnum'
-
     - name: ensure /etc/ansible exists
       file:
         path: /etc/ansible
@@ -282,6 +264,13 @@
         docker_image_tag: "{{ build_image_tag if need_build_image else (zuul.branch | basename) ~ docker_image_tag_suffix }}"
         docker_image_prefix: "{{ 'primary:4000/lokolla/' if need_build_image else 'kolla/' }}"
 
+    # NOTE(yoctozepto): k-a octavia-certificates should run before k-a bootstrap-servers
+    # because the latter hijacks /etc/kolla permissions (due to same directory on the
+    # same host being used by both)
+    - name: create TLS certificates for octavia
+      command: kolla-ansible octavia-certificates
+      when: scenario == 'magnum'
+
     # NOTE(mgoddard): We are using the script module here and later to ensure
     # we use the local copy of these scripts, rather than the one on the remote
     # host, which could be checked out to a previous release (in an upgrade

tools/kolla-ansible

@@ -121,29 +121,30 @@ Environment variables:
     EXTRA_OPTS                         Additional arguments to pass to ansible-playbook
 
 Commands:
-    prechecks           Do pre-deployment checks for hosts
+    prechecks            Do pre-deployment checks for hosts
-    check               Do post-deployment smoke tests
+    check                Do post-deployment smoke tests
-    mariadb_recovery    Recover a completely stopped mariadb cluster
+    mariadb_recovery     Recover a completely stopped mariadb cluster
-    mariadb_backup      Take a backup of MariaDB databases
+    mariadb_backup       Take a backup of MariaDB databases
-                            --full (default)
+                             --full (default)
-                            --incremental
+                             --incremental
-    bootstrap-servers   Bootstrap servers with kolla deploy dependencies
+    bootstrap-servers    Bootstrap servers with kolla deploy dependencies
-    destroy             Destroy Kolla containers, volumes and host configuration
+    destroy              Destroy Kolla containers, volumes and host configuration
-                            --include-images to also destroy Kolla images
+                             --include-images to also destroy Kolla images
-                            --include-dev to also destroy dev mode repos
+                             --include-dev to also destroy dev mode repos
-    deploy              Deploy and start all kolla containers
+    deploy               Deploy and start all kolla containers
-    deploy-bifrost      Deploy and start bifrost container
+    deploy-bifrost       Deploy and start bifrost container
-    deploy-servers      Enroll and deploy servers with bifrost
+    deploy-servers       Enroll and deploy servers with bifrost
-    deploy-containers   Only deploy and start containers (no config updates or bootstrapping)
+    deploy-containers    Only deploy and start containers (no config updates or bootstrapping)
-    post-deploy         Do post deploy on deploy node
+    post-deploy          Do post deploy on deploy node
-    pull                Pull all images for containers (only pulls, no running container changes)
+    pull                 Pull all images for containers (only pulls, no running container changes)
-    reconfigure         Reconfigure OpenStack service
+    reconfigure          Reconfigure OpenStack service
-    stop                Stop Kolla containers
+    stop                 Stop Kolla containers
-    certificates        Generate self-signed certificate for TLS *For Development Only*
+    certificates         Generate self-signed certificate for TLS *For Development Only*
-    upgrade             Upgrades existing OpenStack Environment
+    octavia-certificates Generate certificates for octavia deployment
-    upgrade-bifrost     Upgrades an existing bifrost container
+    upgrade              Upgrades existing OpenStack Environment
-    genconfig           Generate configuration files for enabled OpenStack services
+    upgrade-bifrost      Upgrades an existing bifrost container
-    prune-images        Prune orphaned Kolla images
+    genconfig            Generate configuration files for enabled OpenStack services
+    prune-images         Prune orphaned Kolla images
 EOF
 }
 
@@ -179,6 +180,7 @@ pull
 reconfigure
 stop
 certificates
+octavia-certificates
 upgrade
 upgrade-bifrost
 genconfig
@@ -431,6 +433,10 @@ EOF
         ACTION="Generate TLS Certificates"
         PLAYBOOK="${BASEDIR}/ansible/certificates.yml"
         ;;
+(octavia-certificates)
+        ACTION="Generate octavia Certificates"
+        PLAYBOOK="${BASEDIR}/ansible/octavia-certificates.yml"
+        ;;
 (genconfig)
         ACTION="Generate configuration files for enabled OpenStack services"
         EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=config"

zuul.d/base.yaml

@@ -131,7 +131,7 @@
     parent: kolla-ansible-base
     voting: false
     files:
-      - ^ansible/roles/(designate|magnum|octavia)/
+      - ^ansible/roles/(designate|magnum|octavia|octavia-certificates)/
       - ^tests/test-dashboard.sh
       - ^tests/test-magnum.sh
     vars: