Improve network configuration & external network support

- Support multiple OVS bridges - one for provisioning and another optional
  bridge for external network access.
- Use a more consistent naming scheme for OVS bridges and veth pairs
- Add an inspection network used for inspecting the workload hosts.

ansible/group_vars/all/network

@@ -22,3 +22,24 @@ storage_net_name: 'storage_net'
 
 # Name of the network used to carry storage management traffic.
 storage_mgmt_net_name: 'storage_mgmt_net'
+
+# Name of the network used to perform hardware introspection on the bare metal
+# workload hosts.
+inspection_net_name: 'inspection_net'
+
+###############################################################################
+# Network interface naming conventions.
+
+# Suffix for Open vSwitch bridge names.
+network_bridge_suffix_ovs: '-ovs'
+
+# Prefix for virtual patch interface names.
+network_patch_prefix: 'p-'
+
+# Suffix for virtual patch link interface names when connected towards the
+# physical interface.
+network_patch_suffix_phy: '-phy'
+
+# Suffix for virtual patch link interface names when connected towards the
+# OVS bridge.
+network_patch_suffix_ovs: '-ovs'

ansible/group_vars/controllers/network

@@ -18,12 +18,3 @@ controller_default_network_interfaces: >
 
 # List of extra networks to which controller nodes are attached.
 controller_extra_network_interfaces: []
-
-###############################################################################
-# Kolla networking.
-
-# Name of the Neutron OVS bridge for the provisioning network.
-neutron_bridge_name: "br-ex"
-
-# External network interface for Neutron.
-neutron_external_interface: "{{ 'patch-' ~ neutron_bridge_name }}"

ansible/kolla-ansible.yml

@@ -2,30 +2,163 @@
 - name: Ensure Kolla Ansible is configured
   hosts: config-mgmt
   vars:
-    controller_host: "{{ groups['controllers'][0] | default() }}"
+    # We need to reference configuration for the controller and seed nodes.
-    seed_host: "{{ groups['seed'][0] | default() }}"
+    # We pick the first host from each group for this. It is possible that at
+    # this point these groups have no hosts in, and we should handle that case
+    # gracefully.
+    # NOTE: Per-host configuration is not currently supported.
+    controller_host: "{{ groups['controllers'][0] }}"
+    seed_host: "{{ groups['seed'][0] }}"
   pre_tasks:
-    - name: Check whether a Kolla extra globals configuration file exists
+    # Configuration of extra user-provided Kolla globals.
-      stat:
+    - block:
-        path: "{{ kayobe_config_path ~ '/kolla/globals.yml' }}"
+        - name: Check whether a Kolla extra globals configuration file exists
-      register: globals_stat
+          stat:
+            path: "{{ kayobe_config_path ~ '/kolla/globals.yml' }}"
+          register: globals_stat
+
+        - name: Read the Kolla extra globals configuration file
+          set_fact:
+            kolla_extra_globals: "{{ lookup('template', kayobe_config_path ~ '/kolla/globals.yml') | from_yaml }}"
+          when: globals_stat.stat.exists
+      tags:
+        - config
+
+    # Configuration and validation of controller networking.
+    - block:
+        - name: Set facts containing the VIP addresses and FQDNs
+          set_fact:
+            kolla_internal_vip_address: "{{ internal_net_name | net_vip_address }}"
+            kolla_internal_fqdn: "{{ internal_net_name | net_fqdn or internal_net_name | net_vip_address }}"
+            kolla_external_vip_address: "{{ external_net_name | net_vip_address }}"
+            kolla_external_fqdn: "{{ external_net_name | net_fqdn or external_net_name | net_vip_address }}"
+          when: "{{ kolla_enable_haproxy | bool }}"
+
+        - name: Set facts containing the VIP addresses and FQDNs
+          set_fact:
+            kolla_internal_vip_address: "{{ internal_net_name | net_ip(controller_host) }}"
+            kolla_internal_fqdn: "{{ internal_net_name | net_ip(controller_host) }}"
+            kolla_external_vip_address: "{{ external_net_name | net_ip(controller_host) }}"
+            kolla_external_fqdn: "{{ external_net_name | net_ip(controller_host) }}"
+          when:
+            - "{{ not kolla_enable_haproxy | bool }}"
+
+        - name: Set facts containing the controller network interfaces
+          set_fact:
+            kolla_network_interface: "{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_external_vip_interface: "{{ external_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_api_interface: "{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_storage_interface: "{{ storage_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_cluster_interface: "{{ storage_mgmt_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_provision_interface: "{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}"
+            kolla_inspector_dnsmasq_interface: "{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}"
+
+        - name: Set facts containing the Neutron bridge and interface names for the provisioning network
+          set_fact:
+            kolla_neutron_bridge_names:
+              - "{{ provision_wl_net_name | net_interface(controller_host) ~ network_bridge_suffix_ovs }}"
+            kolla_neutron_external_interfaces:
+              - "{{ network_patch_prefix ~ provision_wl_net_name | net_interface(controller_host) ~ network_patch_suffix_ovs }}"
+
+        - name: Update facts containing the Neutron bridge and interface names for the external network
+          set_fact:
+            kolla_neutron_bridge_names: >
+              {{ kolla_neutron_bridge_names +
+                 [external_net_name | net_interface(controller_host) ~ network_bridge_suffix_ovs] }}
+            kolla_neutron_external_interfaces: >
+              {{ kolla_neutron_external_interfaces +
+                 [network_patch_prefix ~ external_net_name | net_interface(controller_host) ~ network_patch_suffix_ovs] }}
+          when:
+            - "{{ provision_wl_net_name != external_net_name }}"
+
+        - name: Validate controller Kolla Ansible network configuration
+          fail:
+            msg: >
+              The Kolla Ansible variable {{ item.var_name }}
+              ({{ item.description }}) is invalid. Value:
+              "{{ hostvars[inventory_hostname][item.var_name] | default('<undefined>') }}".
+          when:
+            - "{{ item.required | bool }}"
+            - "{{ hostvars[inventory_hostname][item.var_name] is not defined or not hostvars[inventory_hostname][item.var_name] }}"
+          with_items:
+            - var_name: "kolla_internal_vip_address"
+              description: "Internal API VIP address"
+              required: True
+            - var_name: "kolla_internal_fqdn"
+              description: "Internal API Fully Qualified Domain Name (FQDN)"
+              required: True
+            - var_name: "kolla_external_vip_address"
+              description: "external API VIP address"
+              required: True
+            - var_name: "kolla_external_fqdn"
+              description: "External API Fully Qualified Domain Name (FQDN)"
+              required: True
+            - var_name: "kolla_api_interface"
+              description: "API network interface name"
+              required: True
+            - var_name: "kolla_external_vip_interface"
+              description: "External network interface name"
+              required: True
+            - var_name: "kolla_provision_interface"
+              description: "Bare metal provisioning network interface name"
+              required: "{{ kolla_enable_ironic }}"
+            - var_name: "kolla_inspector_dnsmasq_interface"
+              description: "Bare metal introspection network interface name"
+              required: "{{ kolla_enable_ironic }}"
+            - var_name: "kolla_neutron_bridge_names"
+              description: "List of Neutron bridge names"
+              required: True
+            - var_name: "kolla_neutron_external_interfaces"
+              description: "List of Neutron interface names"
+              required: True
+
+        - name: Validate controller Kolla Ansible Neutron bridge and interface configuration
+          fail:
+            msg: >
+              The Kolla Ansible variable {{ item.0.var_name }}
+              ({{ item.0.description }}) is invalid. Value:
+              "{{ item.1 | default('<undefined>') }}".
+          when:
+            - "{{ item.0.required | bool }}"
+            - "{{ item.1 is not defined or not item.1 }}"
+          with_subelements:
+            - - var_name: "kolla_neutron_bridge_names"
+                value: "{{ kolla_neutron_bridge_names }}"
+                description: "List of Neutron bridge names"
+                required: True
+              - var_name: "kolla_neutron_external_interfaces"
+                value: "{{ kolla_neutron_external_interfaces }}"
+                description: "List of Neutron interface names"
+                required: True
+            - value
+      when: "{{ controller_host is defined }}"
+      tags:
+        - config
+        - config-validation
+
+    # Configuration and validation of seed networking.
+    - block:
+        - name: Set facts containing the seed network interfaces
+          set_fact:
+            kolla_bifrost_network_interface: "{{ provision_oc_net_name | net_interface(seed_host) | replace('-', '_') }}"
+
+        - name: Validate seed Kolla Ansible network configuration
+          fail:
+            msg: >
+              The Kolla Ansible variable {{ item.var_name }}
+              ({{ item.description }}) is invalid. Value:
+              "{{ hostvars[inventory_hostname][item.var_name] | default('<undefined>') }}".
+          when:
+            - "{{ item.required | bool }}"
+            - "{{ hostvars[inventory_hostname][item.var_name] is not defined or not hostvars[inventory_hostname][item.var_name] }}"
+          with_items:
+            - var_name: "kolla_bifrost_network_interface"
+              description: "Bifrost network interface name"
+              required: True
+      when: "{{ seed_host is defined }}"
+      tags:
+        - config
+        - config-validation
 
-    - name: Read the Kolla extra globals configuration file
-      set_fact:
-        kolla_extra_globals: "{{ lookup('template', kayobe_config_path ~ '/kolla/globals.yml') | from_yaml }}"
-      when: globals_stat.stat.exists
   roles:
     - role: kolla-ansible
-      kolla_internal_vip_address: "{% if kolla_enable_haproxy | bool %}{{ internal_net_name | net_vip_address }}{% elif controller_host %}{{ internal_net_name | net_ip(controller_host) }}{% endif %}"
-      kolla_internal_fqdn: "{% if kolla_enable_haproxy | bool %}{{ internal_net_name | net_fqdn or kolla_internal_vip_address }}{% elif controller_host %}{{ internal_net_name | net_ip(controller_host) }}{% endif %}"
-      kolla_external_vip_address: "{% if kolla_enable_haproxy | bool %}{{ external_net_name | net_vip_address }}{% elif controller_host %}{{ external_net_name | net_ip(controller_host) }}{% endif %}"
-      kolla_external_fqdn: "{% if kolla_enable_haproxy | bool %}{{ external_net_name | net_fqdn or kolla_external_vip_address }}{% elif controller_host %}{{ external_net_name | net_ip(controller_host) }}{% endif %}"
-      kolla_network_interface: "{% if controller_host %}{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"
-      kolla_external_vip_interface: "{% if controller_host %}{{ external_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"
-      kolla_api_interface: "{% if controller_host %}{{ internal_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"
-      kolla_storage_interface: "{% if controller_host %}{{ storage_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"
-      kolla_cluster_interface: "{% if controller_host %}{{ storage_mgmt_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"
-      kolla_neutron_bridge_name: "{% if controller_host %}{{ hostvars[controller_host]['neutron_bridge_name'] }}{% endif %}"
-      kolla_neutron_external_interface: "{% if controller_host %}{{ hostvars[controller_host]['neutron_external_interface'] }}{% endif %}"
-      kolla_bifrost_network_interface: "{% if seed_host %}{{ provision_oc_net_name | net_interface(seed_host) | replace('-', '_') }}{% endif %}"
-      kolla_provision_interface: "{% if controller_host %}{{ provision_wl_net_name | net_interface(controller_host) | replace('-', '_') }}{% endif %}"

ansible/network.yml

@@ -2,6 +2,26 @@
 - name: Ensure networking is configured
   hosts: seed:controllers
   pre_tasks:
+    - block:
+        - name: Validate network interface configuration
+          fail:
+            msg: >
+              Network interface validation failed - no interface configured for
+              {{ item }}. This should be configured via '{{ item }}_interface'.
+          with_items: "{{ network_interfaces | net_select_ethers | list }}"
+          when: "{{ not item | net_interface }}"
+
+        - name: Validate bridge interface configuration
+          fail:
+            msg: >
+              Bridge interface validation failed - no interface configured for
+              {{ item }}. This should be configured via '{{ item }}_interface'.
+          with_items: "{{ network_interfaces | net_select_bridges | list }}"
+          when: "{{ not item | net_interface }}"
+      tags:
+        - config
+        - config-validation
+
     - name: Ensure NetworkManager is disabled
       service:
         name: NetworkManager
@@ -17,6 +37,8 @@
   roles:
     - role: ahuffman.resolv
       become: True
+      tags:
+        - config
 
     - role: MichaelRigart.interfaces
       interfaces_ether_interfaces: >
@@ -30,18 +52,38 @@
            map('net_bridge_obj') |
            list }}
       become: True
+      tags:
+        - config
 
-- name: Ensure controller workload provisioning network is configured
+- name: Ensure controller workload OVS patch links exist
   hosts: controllers
   roles:
     # Configure a virtual ethernet patch link to connect the workload provision
     # network bridge to the Neutron OVS bridge.
     - role: veth
       veth_interfaces:
-        - device: "{{ 'patch-' ~ provision_wl_net_name | net_interface }}"
+        - device: "{{ network_patch_prefix ~ provision_wl_net_name | net_interface ~ network_patch_suffix_phy }}"
           bootproto: "static"
           bridge: "{{ provision_wl_net_name | net_interface }}"
-          peer_device: "{{ 'patch-' ~ neutron_bridge_name }}"
+          peer_device: "{{ network_patch_prefix ~ provision_wl_net_name | net_interface ~ network_patch_suffix_ovs }}"
           peer_bootproto: "static"
           onboot: yes
       when: "{{ provision_wl_net_name in network_interfaces }}"
+      tags:
+        - config
+
+    # Configure a virtual ethernet patch link to connect the external network
+    # bridge to the Neutron OVS bridge.
+    - role: veth
+      veth_interfaces:
+        - device: "{{ network_patch_prefix ~ external_net_name | net_interface ~ network_patch_suffix_phy }}"
+          bootproto: "static"
+          bridge: "{{ external_net_name | net_interface }}"
+          peer_device: "{{ network_patch_prefix ~ external_net_name | net_interface ~ network_patch_suffix_ovs }}"
+          peer_bootproto: "static"
+          onboot: yes
+      when:
+        - "{{ external_net_name in network_interfaces }}"
+        - "{{ external_net_name != provision_wl_net_name }}"
+      tags:
+        - config

ansible/roles/kolla-ansible/defaults/main.yml

@@ -81,10 +81,11 @@ kolla_cluster_interface:
 # though an IP address can exist on this interface, it will be unusable in most
 # configurations. It is recommended this interface not be configured with any IP
 # addresses for that reason.
-kolla_neutron_external_interface:
+# This should be a list of names of interfaces.
+kolla_neutron_external_interfaces: []
 
-# Name of the Neutron external bridge.
+# List of names of Neutron bridges.
-kolla_neutron_bridge_name:
+kolla_neutron_bridge_names: []
 
 # This is the interface to use for Bifrost bare metal provisioning of the
 # control plane.
@@ -94,6 +95,8 @@ kolla_bifrost_network_interface:
 # standard kolla variable.
 kolla_provision_interface:
 
+# Interface on which the inspector dnsmasq service listens.
+kolla_inspector_dnsmasq_interface:
 
 ####################
 # TLS options

ansible/roles/kolla-ansible/templates/globals.yml.j2

@@ -83,10 +83,10 @@ cluster_interface: "{{ kolla_cluster_interface }}"
 # though an IP address can exist on this interface, it will be unusable in most
 # configurations. It is recommended this interface not be configured with any IP
 # addresses for that reason.
-neutron_external_interface: "{{ kolla_neutron_external_interface }}"
+neutron_external_interface: "{{ kolla_neutron_external_interfaces | join(',') }}"
 
 # Name of the Neutron external bridge.
-neutron_bridge_name: "{{ kolla_neutron_bridge_name }}"
+neutron_bridge_name: "{{ kolla_neutron_bridge_names | join(',') }}"
 
 # This is the interface to use for Bifrost bare metal provisioning of the
 # control plane.
@@ -96,6 +96,9 @@ bifrost_network_interface: "{{ kolla_bifrost_network_interface }}"
 # standard kolla variable.
 provision_interface: "{{ kolla_provision_interface }}"
 
+# Interface on which the inspector dnsmasq service listens.
+ironic_dnsmasq_interface: "{{ kolla_inspector_dnsmasq_interface }}"
+
 # Valid options are [ openvswitch, linuxbridge ]
 neutron_plugin_agent: "openvswitch"
 

etc/kayobe/networks.yml

@@ -25,6 +25,10 @@
 # Name of the network used to carry storage management traffic.
 #storage_mgmt_net_name:
 
+# Name of the network used to perform hardware introspection on the bare metal
+# workload hosts.
+#inspection_net_name:
+
 ###############################################################################
 # Network definitions.
 
@@ -74,6 +78,23 @@
 # storage_mgmt_net_gateway:
 # storage_mgmt_net_vlan:
 
+###############################################################################
+# Network virtual patch link configuration.
+
+# Suffix for Open vSwitch bridge names.
+#network_bridge_suffix_ovs:
+
+# Prefix for virtual patch interface names.
+#network_patch_prefix:
+
+# Suffix for virtual patch link interface names when connected towards the
+# physical interface.
+#network_patch_suffix_phy:
+
+# Suffix for virtual patch link interface names when connected towards the
+# OVS bridge.
+#network_patch_suffix_ovs:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes