From 781d84658ddfdf98307a62830d5a800ad00164ef Mon Sep 17 00:00:00 2001
From: Sean Mooney <sean.k.mooney@intel.com>
Date: Thu, 6 Apr 2017 13:21:09 +0000
Subject: [PATCH] introduce playbook to ovs with dpdk
- This change creates a new role to
support deploying ovs with dpdk.
- This change introduces an enable_ovs_dpdk
variable to enable ovs with dpdk as part of
the deploy action.
- This change extends the ovs-dpdkctl.sh tool
to correctly deploy ovs-dpdk on Ubuntu and CentOS
hosts.
- This change extends the cleanup-host tool
to correctly uninstall ovs-dpdkctl.sh tool and its
systemd files if present on the host.
- This change automatically configures userspace
kernel drivers.
partial-implementes: bp/ovs-dpdk
Change-Id: I55858d81df437e8258705b75426f61ab9b907c7d
---
ansible/group_vars/all.yml | 1 +
ansible/roles/ovs-dpdk/defaults/main.yml | 86 +++++++++++
ansible/roles/ovs-dpdk/handlers/main.yml | 108 ++++++++++++++
ansible/roles/ovs-dpdk/tasks/check.yml | 1 +
ansible/roles/ovs-dpdk/tasks/config.yml | 60 ++++++++
ansible/roles/ovs-dpdk/tasks/deploy.yml | 5 +
ansible/roles/ovs-dpdk/tasks/main.yml | 2 +
ansible/roles/ovs-dpdk/tasks/prechecks.yml | 1 +
ansible/roles/ovs-dpdk/tasks/pull.yml | 11 ++
ansible/roles/ovs-dpdk/tasks/reconfigure.yml | 2 +
ansible/roles/ovs-dpdk/tasks/upgrade.yml | 2 +
.../ovs-dpdk/templates/ovsdpdk-db.json.j2 | 4 +
.../templates/ovsdpdk-vswitchd.json.j2 | 4 +
ansible/site.yml | 10 +-
etc/kolla/globals.yml | 1 +
.../notes/ovs-dpdk-a48404777d3836a3.yaml | 43 ++++++
tools/cleanup-host | 8 +-
tools/kolla-ansible | 2 +-
tools/ovs-dpdkctl.sh | 137 ++++++++++++++++--
19 files changed, 474 insertions(+), 14 deletions(-)
create mode 100644 ansible/roles/ovs-dpdk/defaults/main.yml
create mode 100644 ansible/roles/ovs-dpdk/handlers/main.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/check.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/config.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/deploy.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/main.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/prechecks.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/pull.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/reconfigure.yml
create mode 100644 ansible/roles/ovs-dpdk/tasks/upgrade.yml
create mode 100644 ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2
create mode 100644 ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2
create mode 100644 releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 992eb4b2d9..b517f17c21 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -390,6 +390,7 @@ enable_nova_serialconsole_proxy: "no"
enable_octavia: "no"
enable_opendaylight: "no"
enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' | bool }}"
+enable_ovs_dpdk: "no"
enable_osprofiler: "no"
enable_panko: "no"
enable_qdrouterd: "no"
diff --git a/ansible/roles/ovs-dpdk/defaults/main.yml b/ansible/roles/ovs-dpdk/defaults/main.yml
new file mode 100644
index 0000000000..4447522ef7
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/defaults/main.yml
@@ -0,0 +1,86 @@
+---
+####################
+# Services
+####################
+
+ovsdpdk_services:
+ ovsdpdk-db:
+ container_name: "ovsdpdk_db"
+ image: "{{ ovsdpdk_db_image_full }}"
+ enabled: "{{ neutron_plugin_agent == 'openvswitch' }}"
+ host_in_groups: >-
+ {{
+ inventory_hostname in groups['compute']
+ or (enable_manila | bool and inventory_hostname in groups['manila-share'])
+ or inventory_hostname in groups['neutron-dhcp-agent']
+ or inventory_hostname in groups['neutron-l3-agent']
+ or inventory_hostname in groups['neutron-metadata-agent']
+ or inventory_hostname in groups['neutron-vpnaas-agent']
+ }}
+ volumes:
+ - "{{ node_config_directory }}/ovsdpdk-db/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "/run:/run:shared"
+ - "kolla_logs:/var/log/kolla/"
+ - "ovsdpdk_db:/var/lib/openvswitch/"
+ ovsdpdk-vswitchd:
+ container_name: "ovsdpdk_vswitchd"
+ image: "{{ ovsdpdk_vswitchd_image_full }}"
+ enabled: "{{ neutron_plugin_agent == 'openvswitch' }}"
+ privileged: True
+ host_in_groups: >-
+ {{
+ inventory_hostname in groups['compute']
+ or (enable_manila | bool and inventory_hostname in groups['manila-share'])
+ or inventory_hostname in groups['neutron-dhcp-agent']
+ or inventory_hostname in groups['neutron-l3-agent']
+ or inventory_hostname in groups['neutron-metadata-agent']
+ or inventory_hostname in groups['neutron-vpnaas-agent']
+ }}
+ volumes:
+ - "{{ node_config_directory }}/ovsdpdk-vswitchd/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "/lib/modules:/lib/modules:ro"
+ - "/run:/run:shared"
+ - "/dev:/dev:shared"
+ - "kolla_logs:/var/log/kolla/"
+
+####################
+# OVS
+####################
+ovs_datapath: "netdev"
+ovs_bridge_mappings: "{% for bridge in neutron_bridge_name.split(',') %}physnet{{ loop.index0 + 1 }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %}"
+ovs_port_mappings: "{% for bridge in neutron_bridge_name.split(',') %} {{ neutron_external_interface.split(',')[loop.index0] }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %}"
+dpdk_tunnel_interface : "{{neutron_external_interface}}"
+dpdk_tunnel_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['address'] }}"
+tunnel_interface_network: "{{ hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['network']}}/{{hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['netmask']}}"
+tunnel_interface_cidr: "{{dpdk_tunnel_interface_address}}/{{ tunnel_interface_network | ipaddr('prefix') }}"
+ovs_cidr_mappings: "{% if neutron_bridge_name.split(',')|length != 1 %} {neutron_bridge_name.split(',')[0]}:{{ tunnel_interface_cidr }} {% else %} {{ neutron_bridge_name }}:{{ tunnel_interface_cidr }} {% endif %}"
+ovs_mem_channels: 4
+ovs_socket_mem: 1024
+ovs_hugepage_mountpoint: /dev/hugepages
+
+# ovs <2.7 required dpdk phyical port names to be index
+# in pci adress order as dpdkX where X is the index
+# ovs>=2.7 allows arbitray names but the pci adress
+# must be set in a new dpdkdev-opt field
+# valid values are indexed or named.
+ovs_physical_port_policy: named
+
+project_name: "ovs"
+neutron_bridge_name: "br_dpdk"
+
+
+####################
+# Docker
+####################
+ovsdpdk_install_type: "{{ kolla_install_type }}"
+ovsdpdk_tag: "{{ openstack_release }}"
+
+ovsdpdk_db_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ ovsdpdk_install_type }}-ovsdpdk-db"
+ovsdpdk_db_tag: "{{ ovsdpdk_tag }}"
+ovsdpdk_db_image_full: "{{ ovsdpdk_db_image }}:{{ ovsdpdk_db_tag }}"
+
+ovsdpdk_vswitchd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ ovsdpdk_install_type }}-ovsdpdk-vswitchd"
+ovsdpdk_vswitchd_tag: "{{ ovsdpdk_tag }}"
+ovsdpdk_vswitchd_image_full: "{{ ovsdpdk_vswitchd_image }}:{{ ovsdpdk_vswitchd_tag }}"
diff --git a/ansible/roles/ovs-dpdk/handlers/main.yml b/ansible/roles/ovs-dpdk/handlers/main.yml
new file mode 100644
index 0000000000..4401940c3b
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/handlers/main.yml
@@ -0,0 +1,108 @@
+---
+- name: Restart ovsdpdk-db container
+ vars:
+ service_name: "ovsdpdk-db"
+ service: "{{ ovsdpdk_services[service_name] }}"
+ config_json: "{{ ovsdpdk_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ ovsdpdk_db_container: "{{ check_ovsdpdk_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ when:
+ - action != "config"
+ - service.enabled | bool
+ - service.host_in_groups | bool
+ - config_json | changed
+ or ovsdpdk_db_container | changed
+ notify:
+ - Waiting the ovs db service to be ready
+ - Ensuring ovsdpdk bridges are properly setup indexed
+ - Restart ovsdpdk-vswitchd container
+ - Ensuring ovsdpdk bridges are properly setup named
+ - wait for dpdk tunnel ip
+ - ovs-dpdk gather facts
+
+- name: Waiting the ovs db service to be ready
+ vars:
+ service_name: "ovsdpdk-db"
+ service: "{{ ovsdpdk_services[service_name] }}"
+ command: docker exec "{{ service.container_name }}" ovs-vsctl --no-wait show
+ register: check_result
+ until: check_result | success
+ changed_when: False
+ retries: 30
+ delay: 2
+
+- name: Ensuring ovsdpdk bridges are properly setup indexed
+ vars:
+ service_name: "ovsdpdk-db"
+ service: "{{ ovsdpdk_services[service_name] }}"
+ command: "docker exec -e \"CONFIG_FILE={{ container_config_directory }}/ovs-dpdkctl.conf\" {{ service.container_name }} {{ container_config_directory }}/ovs-dpdkctl.sh init"
+ register: status
+ changed_when: status.stdout.find('changed') != -1
+ when:
+ - (inventory_hostname in groups['compute']
+ or (enable_manila | bool and inventory_hostname in groups['manila-share'])
+ or inventory_hostname in groups['neutron-dhcp-agent']
+ or inventory_hostname in groups['neutron-l3-agent']
+ or inventory_hostname in groups['neutron-metadata-agent']
+ or inventory_hostname in groups['neutron-vpnaas-agent'])
+ - ovs_physical_port_policy == 'indexed'
+
+- name: Restart ovsdpdk-vswitchd container
+ vars:
+ service_name: "ovsdpdk-vswitchd"
+ service: "{{ ovsdpdk_services[service_name] }}"
+ config_json: "{{ ovsdpdk_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ ovsdpdk_vswitchd_container: "{{ check_ovsdpdk_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ privileged: "{{ service.privileged | default(True) }}"
+ when:
+ - action != "config"
+ - service.enabled | bool
+ - service.host_in_groups | bool
+ - config_json | changed
+ or ovsdpdk_vswitchd_container | changed
+
+
+- name: Ensuring ovsdpdk bridges are properly setup named
+ vars:
+ service_name: "ovsdpdk-db"
+ service: "{{ ovsdpdk_services[service_name] }}"
+ command: "docker exec -e \"CONFIG_FILE={{ container_config_directory }}/ovs-dpdkctl.conf\" {{ service.container_name }} {{ container_config_directory }}/ovs-dpdkctl.sh init"
+ register: status
+ changed_when: status.stdout.find('changed') != -1
+ when:
+ - (inventory_hostname in groups['compute']
+ or (enable_manila | bool and inventory_hostname in groups['manila-share'])
+ or inventory_hostname in groups['neutron-dhcp-agent']
+ or inventory_hostname in groups['neutron-l3-agent']
+ or inventory_hostname in groups['neutron-metadata-agent']
+ or inventory_hostname in groups['neutron-vpnaas-agent'])
+ - ovs_physical_port_policy == 'named'
+
+
+- name: wait for dpdk tunnel ip
+ wait_for:
+ host: "{{ dpdk_tunnel_interface_address }}"
+ timeout: 300
+ sleep: 5
+
+- name: ovs-dpdk gather facts
+ setup:
+ filter: ansible_*
+ when:
+ - (inventory_hostname in groups['compute']
+ or (enable_manila | bool and inventory_hostname in groups['manila-share'])
+ or inventory_hostname in groups['neutron-dhcp-agent']
+ or inventory_hostname in groups['neutron-l3-agent']
+ or inventory_hostname in groups['neutron-metadata-agent']
+ or inventory_hostname in groups['neutron-vpnaas-agent'])
diff --git a/ansible/roles/ovs-dpdk/tasks/check.yml b/ansible/roles/ovs-dpdk/tasks/check.yml
new file mode 100644
index 0000000000..ed97d539c0
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/check.yml
@@ -0,0 +1 @@
+---
diff --git a/ansible/roles/ovs-dpdk/tasks/config.yml b/ansible/roles/ovs-dpdk/tasks/config.yml
new file mode 100644
index 0000000000..bfb7b15bff
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/config.yml
@@ -0,0 +1,60 @@
+---
+- name: Ensuring config directories exist
+ file:
+ path: "{{ node_config_directory }}/{{ item.key }}"
+ state: "directory"
+ recurse: yes
+ when:
+ - item.value.enabled | bool
+ - item.value.host_in_groups | bool
+ with_dict: "{{ ovsdpdk_services }}"
+
+- name: Copying over config.json files for services
+ template:
+ src: "{{ item.key }}.json.j2"
+ dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
+ register: ovsdpdk_config_jsons
+ when:
+ - item.value.enabled | bool
+ - item.value.host_in_groups | bool
+ with_dict: "{{ ovsdpdk_services }}"
+ notify:
+ - "Restart {{ item.key }} container"
+
+- name: Copying ovs-dpdkctl tool
+ copy:
+ src: ../tools/ovs-dpdkctl.sh
+ dest: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.sh"
+ mode: 0777
+
+- name: Install ovs-dpdkctl service and config
+ become: True
+ command: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.sh install"
+ environment:
+ CONFIG_FILE: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.conf"
+ bridge_mappings: "{{ ovs_bridge_mappings }}"
+ port_mappings: "{{ ovs_port_mappings }}"
+ cidr_mappings: "{{ ovs_cidr_mappings }}"
+ ovs_mem_channels: "{{ ovs_mem_channels }}"
+ ovs_socket_mem: "{{ ovs_socket_mem }}"
+ hugepage_mountpoint: "{{ ovs_hugepage_mountpoint }}"
+ ovs_physical_port_policy: "{{ ovs_physical_port_policy }}"
+
+
+# TODO check the environment change
+- name: Check ovs containers
+ kolla_docker:
+ action: "compare_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ item.value.container_name }}"
+ image: "{{ item.value.image }}"
+ privileged: "{{ item.value.privileged | default(False) }}"
+ volumes: "{{ item.value.volumes }}"
+ register: check_ovsdpdk_containers
+ when:
+ - action != "config"
+ - item.value.enabled | bool
+ - item.value.host_in_groups | bool
+ with_dict: "{{ ovsdpdk_services }}"
+ notify:
+ - "Restart {{ item.key }} container"
diff --git a/ansible/roles/ovs-dpdk/tasks/deploy.yml b/ansible/roles/ovs-dpdk/tasks/deploy.yml
new file mode 100644
index 0000000000..5aac9f5a7f
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/deploy.yml
@@ -0,0 +1,5 @@
+---
+- include: config.yml
+
+- name: Flush Handlers
+ meta: flush_handlers
diff --git a/ansible/roles/ovs-dpdk/tasks/main.yml b/ansible/roles/ovs-dpdk/tasks/main.yml
new file mode 100644
index 0000000000..b017e8b4ad
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"
diff --git a/ansible/roles/ovs-dpdk/tasks/prechecks.yml b/ansible/roles/ovs-dpdk/tasks/prechecks.yml
new file mode 100644
index 0000000000..ed97d539c0
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/prechecks.yml
@@ -0,0 +1 @@
+---
diff --git a/ansible/roles/ovs-dpdk/tasks/pull.yml b/ansible/roles/ovs-dpdk/tasks/pull.yml
new file mode 100644
index 0000000000..9341fcc4b6
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/pull.yml
@@ -0,0 +1,11 @@
+---
+- name: Pulling ovs-dpdk images
+ kolla_docker:
+ action: "pull_image"
+ common_options: "{{ docker_common_options }}"
+ image: "{{ item.value.image }}"
+ when:
+ - item.value.enabled | bool
+ - item.value.host_in_groups | bool
+ with_dict: "{{ ovsdpdk_services }}"
+
diff --git a/ansible/roles/ovs-dpdk/tasks/reconfigure.yml b/ansible/roles/ovs-dpdk/tasks/reconfigure.yml
new file mode 100644
index 0000000000..e078ef1318
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/reconfigure.yml
@@ -0,0 +1,2 @@
+---
+- include: deploy.yml
diff --git a/ansible/roles/ovs-dpdk/tasks/upgrade.yml b/ansible/roles/ovs-dpdk/tasks/upgrade.yml
new file mode 100644
index 0000000000..e078ef1318
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/tasks/upgrade.yml
@@ -0,0 +1,2 @@
+---
+- include: deploy.yml
diff --git a/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2 b/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2
new file mode 100644
index 0000000000..fdeea72ecb
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2
@@ -0,0 +1,4 @@
+{
+ "command": "start-ovsdb-server {{ api_interface_address }}",
+ "config_files": []
+}
diff --git a/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2 b/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2
new file mode 100644
index 0000000000..97308d886b
--- /dev/null
+++ b/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2
@@ -0,0 +1,4 @@
+{
+ "command": "/usr/sbin/ovs-vswitchd unix:/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --log-file=/var/log/kolla/openvswitch/ovs-vswitchd.log",
+ "config_files": []
+}
diff --git a/ansible/site.yml b/ansible/site.yml
index 50cbce1fa0..05435f5926 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -322,7 +322,15 @@
roles:
- { role: openvswitch,
tags: openvswitch,
- when: enable_openvswitch | bool }
+ when: "(enable_openvswitch | bool) and not (enable_ovs_dpdk | bool)"}
+
+- name: Apply role ovs-dpdk
+ hosts:
+ - openvswitch
+ roles:
+ - { role: ovs-dpdk,
+ tags: ovs-dpdk,
+ when: "(enable_openvswitch | bool) and (enable_ovs_dpdk | bool)"}
- name: Apply nova-hyperv role
gather_facts: false
diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml
index f0a4b65963..279a6a1a97 100644
--- a/etc/kolla/globals.yml
+++ b/etc/kolla/globals.yml
@@ -192,6 +192,7 @@ kolla_internal_vip_address: "10.10.10.254"
#enable_octavia: "no"
#enable_opendaylight: "no"
#enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' }}"
+#enable_ovs_dpdk: "no"
#enable_osprofiler: "no"
#enable_panko: "no"
#enable_qdrouterd: "no"
diff --git a/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml b/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml
new file mode 100644
index 0000000000..cb7a031423
--- /dev/null
+++ b/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml
@@ -0,0 +1,43 @@
+---
+prelude: >
+ The Data Plane Development Kit (DPDK) is a collection of userspace
+ libraries and tools that facilitate the development of high-performance
+ userspace networking applications. During the Pike cycle, kolla has been
+ extended to support deploying OpenvSwitch with the dpdk netdev dataplane.
+features:
+ - |
+ A new ovs-dpdk role has been introduced to support deploy OpenvSwitch with
+ the userspace DPDK dataplane. This playbook has been tested on Centos and
+ Ubuntu hosts and introduces a new enable_ovs_dpdk variable to control
+ deployment of OVS with DPDK instead of kernel ovs.
+ - |
+ The ovs-dpdkctl.sh tool has been enhanced to enable automatic loading of
+ DPDK compatible kernel modules.
+ - |
+ The ovs-dpdkctl.sh tool has been extended to configure per host
+ tunnel ip address to enable vxlan and other tunnel networking.
+issues:
+ - |
+ The ovs-dpdk role introduced this release has been tested on centos 7 and
+ ubuntu 16.04 hosts. While both host operating systems can be used to deploy
+ ovs-dpdk containers a conflict exists between the cgroups created by docker
+ and systemd on centos. These conflicts prevent restarting the ovs-dpdk
+ containers once deployed. This issue is not present on Ubuntu and is fixed
+ in newer versions of systemd. Deploying the ovs-dpdk role on centos is not
+ advised in production on CentOS hosts unless you have validated that this
+ issue has been addressed.
+upgrade:
+ - |
+ When upgrading ovs-dpdk it should be noted that this will alway invovle a
+ dataplane outage. Unlike kernel OVS the dataplane for ovs-dpdk executes in
+ the ovs-vswitchd process. As such it is recommended to always evacuate
+ all vm workloads from a node running ovs-dpdk prior to upgrading.
+other:
+ - |
+ Reconfiguring form kernel based OVS to OVS with DPDK has not been tested
+ in this release. Changing ovs datapaths on a deployed node requires
+ neutron config changes and libvirt xml changes for all running instances
+ including a hard reboot of the vm. VMs deployed to an node running ovs-dpdk
+ require hugepage backed memory to function correctly. As such it is not
+ advised to convert an existing compute node to use ovs-dpdk
+ if it currently has VMs deployed.
diff --git a/tools/cleanup-host b/tools/cleanup-host
index 0cf4a69b10..b0917e1e30 100755
--- a/tools/cleanup-host
+++ b/tools/cleanup-host
@@ -56,8 +56,14 @@ if [[ "$glance_file_datadir_volume" != "glance" && -d "$glance_file_datadir_volu
rm -rfv $glance_file_datadir_volume
fi
-echo "Getting folders name..."
FOLDER_PATH="/etc/kolla/"
+
+if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then
+ echo uninstalling ovs-dpdk service files and tools.
+ sudo CONFIG_FILE="$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.conf" $FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh uninstall
+fi
+
+echo "Getting folders name..."
for dir in $FOLDER_PATH*; do
if [ "$dir" == "$FOLDER_PATH""passwords.yml" ] || \
[ "$dir" == "$FOLDER_PATH""globals.yml" ] || \
diff --git a/tools/kolla-ansible b/tools/kolla-ansible
index 53174cae46..63692267a5 100755
--- a/tools/kolla-ansible
+++ b/tools/kolla-ansible
@@ -49,7 +49,7 @@ Commands:
prechecks Do pre-deployment checks for hosts
check Do post-deployment smoke tests
mariadb_recovery Recover a completely stopped mariadb cluster
- bootstrap-servers bootstrap servers with kolla deploy dependencies
+ bootstrap-servers Bootstrap servers with kolla deploy dependencies
destroy Destroy Kolla containers, volumes and host configuration
--include-images to also destroy Kolla images
--include-dev to also destroy dev mode repos
diff --git a/tools/ovs-dpdkctl.sh b/tools/ovs-dpdkctl.sh
index 2b89ad4322..fc5cf2f781 100755
--- a/tools/ovs-dpdkctl.sh
+++ b/tools/ovs-dpdkctl.sh
@@ -8,6 +8,7 @@ fi
FULL_PATH=$(realpath "${BASH_SOURCE[0]}")
CONFIG_FILE=${CONFIG_FILE:-"/etc/default/ovs-dpdk.conf"}
SERVICE_FILE="/etc/systemd/system/ovs-dpdkctl.service"
+BRIDGE_SERVICE_FILE="/etc/systemd/system/ovs-dpdk-bridge.service"
function get_value {
crudini --get $CONFIG_FILE $@
@@ -37,8 +38,12 @@ function del_config {
rm -f $CONFIG_FILE
}
+function is_redhat_family {
+ [[ -e /etc/redhat-release ]] ; echo $?
+}
+
function generate_pciwhitelist {
- local _Whitelist=''
+ _Whitelist=''
for nic in $(list_dpdk_nics); do
address="$(get_value $nic address)"
if [ "$_Whitelist" == '' ]; then
@@ -79,6 +84,7 @@ function gen_config {
set_value ovs ovs_socket_mem ${ovs_socket_mem:-"512"}
set_value ovs dpdk_interface_driver ${dpdk_interface_driver:-"uio_pci_generic"}
set_value ovs hugepage_mountpoint ${hugepage_mountpoint:-"/dev/hugepages"}
+ set_value ovs physical_port_policy ${ovs_physical_port_policy:-"named"}
ls -al /sys/class/net/* | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print "\n[" a[n] "]\naddress = " a[n-2] "\ndriver ="}' >> $CONFIG_FILE
@@ -93,8 +99,8 @@ function gen_config {
}
function bind_nic {
- echo $1 > /sys/bus/pci/drivers/$2/bind
echo $2 > /sys/bus/pci/devices/$1/driver_override
+ echo $1 > /sys/bus/pci/drivers/$2/bind
}
function unbind_nic {
@@ -137,7 +143,6 @@ function unbind_nics {
done
}
-
function get_address_by_name {
ls -al /sys/class/net/$1 | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print a[n-2] }'
}
@@ -147,7 +152,7 @@ function get_driver_by_address {
}
function get_port_bridge {
- for pair in $(get_value ovs port port_mappings); do
+ for pair in $(get_value ovs port_mappings); do
nic=`echo $pair | cut -f 1 -d ":"`
if [[ "$nic" == "$1" ]]; then
bridge=`echo $pair | cut -f 2 -d ":"`
@@ -178,11 +183,11 @@ function init_ovs_bridges {
}
function init_ovs_interfaces {
- local pci_port_pairs ==''
+ pci_port_pairs=''
for nic in $(list_dpdk_nics); do
address="$(get_value $nic address)"
if [ "$pci_port_pairs" == '' ]; then
- pci_port_pairs ="$address,$nic"
+ pci_port_pairs="$address,$nic"
else
pci_port_pairs="$pci_port_pairs $address,$nic"
fi
@@ -194,10 +199,13 @@ function init_ovs_interfaces {
nic="$(echo $pair | cut -f 2 -d ",")"
bridge="$(get_port_bridge $nic)"
# ovs 2.6 and older requires dpdkX names, ovs 2.7+ requires dpdk-devargs instead.
- ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \
- -- set Interface "dpdk${dpdk_port_number}" type=dpdk || \
- ovs-vsctl --no-wait --may-exist add-port $bridge $nic \
- -- set Interface $nic type=dpdk options:dpdk-devargs=$addr
+ if [ "$(get_value ovs physical_port_policy)" == "indexed" ]; then
+ ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \
+ -- set Interface "dpdk${dpdk_port_number}" type=dpdk
+ else
+ ovs-vsctl --may-exist add-port $bridge $nic \
+ -- set Interface $nic type=dpdk options:dpdk-devargs=$addr
+ fi
dpdk_port_number=$((dpdk_port_number+1))
done
@@ -210,6 +218,57 @@ function init {
init_ovs_interfaces
}
+function install_network_manager_conf {
+ pair=$(get_value ovs cidr_mappings)
+ bridge=`echo $pair | cut -f 1 -d ":"`
+ cidr=`echo $pair | cut -f 2 -d ":"`
+ ip=`echo $cidr | cut -f 1 -d "/"`
+ prefix=`echo $cidr | cut -f 2 -d "/"`
+ mask=""
+ full_octets=$(expr $prefix / 8)
+ partial_octet=$(expr $prefix % 8)
+
+ for octet in 0 1 2 3 ; do
+ if [[ "$octet" < "$full_octets" ]]; then
+ mask+=255
+ elif [[ "$octet" == "$full_octets" ]]; then
+ mask+=$((256 - 2**(8-$partial_octet)))
+ else
+ mask+=0
+ fi
+ [[ "$octet" < 3 ]] && mask+=.
+ done
+ if [[ is_redhat_family == 0 ]]; then
+ cat << EOF | tee "/etc/sysconfig/network-scripts/ifcfg-$bridge"
+DEVICE=$bridge
+BOOTPROTO=static
+IPADDR=$ip
+NETMASK=$mask
+HOTPLUG=yes
+ONBOOT=yes
+EOF
+install_redhat_bridge_service $bridge
+ else
+ cat << EOF | tee "/etc/network/interfaces.d/$bridge.cfg"
+ auto $bridge
+ iface $bridge inet static
+ address $ip
+ netmask $mask
+EOF
+
+ fi
+}
+
+function uninstall_network_manager_conf {
+ pair=$(get_value ovs cidr_mappings)
+ bridge=`echo $pair | cut -f 1 -d ":"`
+ if [[ is_redhat_family == 0 ]]; then
+ rm -f /etc/sysconfig/network-scripts/ifcfg-$bridge
+ else
+ rm -f /etc/network/interfaces.d/$bridge.cfg
+ fi
+}
+
function install_service {
cat << EOF | tee "$SERVICE_FILE"
@@ -235,13 +294,63 @@ EOF
systemctl enable ovs-dpdkctl
}
+function install_redhat_bridge_service {
+ cat << EOF | tee "$BRIDGE_SERVICE_FILE"
+[Unit]
+Description=configuration service for ovs-dpdk bridge.
+After=docker.service
+After=network.target
+After=ovs-dpdkctl.service
+
+[Service]
+Type=simple
+RemainAfterExit=yes
+ExecStartPre=/bin/bash -c "[[ -e /sys/class/net/$1 ]]"
+ExecStart=/usr/sbin/ifup $1
+ExecStop=/usr/sbin/ifdown $1
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+ systemctl daemon-reload
+ systemctl enable ovs-dpdk-bridge
+}
+
function uninstall_service {
systemctl disable ovs-dpdkctl
rm -f "$SERVICE_FILE"
+ if [ -e "$BRIDGE_SERVICE_FILE" ]; then
+ systemctl disable ovs-dpdk-bridge
+ rm -f "$BRIDGE_SERVICE_FILE"
+ fi
systemctl daemon-reload
}
+function configure_kernel_modules {
+ driver="$(get_value ovs dpdk_interface_driver)"
+ lsmod | grep -ws $driver > /dev/null || modprobe $driver
+ if [[ is_redhat_family == 0 ]]; then
+ [[ ! -e /etc/modules-load.d/${driver}.conf ]] && echo $driver | tee /etc/modules-load.d/${driver}.conf
+ else
+ grep -ws $driver /etc/modules > /dev/null || echo $driver | tee -a /etc/modules
+ fi
+}
+
+function unconfigure_kernel_modules {
+ driver="$(get_value ovs dpdk_interface_driver)"
+ lsmod | grep -ws $driver > /dev/null && rmmod $driver
+ if [[ is_redhat_family == 0 ]] ; then
+ [[ -e /etc/modules-load.d/${driver}.conf ]] && rm -f /etc/modules-load.d/${driver}.conf
+ else
+ grep -ws $driver /etc/modules > /dev/null && sed -e "s/$driver//" -i /etc/modules
+ fi
+}
+
function install {
+ configure_kernel_modules
if [ ! -e "$SERVICE_FILE" ]; then
install_service
fi
@@ -253,6 +362,10 @@ function install {
gen_config
fi
systemctl start ovs-dpdkctl
+ install_network_manager_conf
+ if [[ is_redhat_family == 0 ]]; then
+ systemctl start ovs-dpdk-bridge
+ fi
}
function uninstall {
@@ -260,14 +373,16 @@ function uninstall {
if [ -e "$SERVICE_FILE" ]; then
uninstall_service
fi
+ uninstall_network_manager_conf
+ unconfigure_kernel_modules
if [ -e /bin/ovs-dpdkctl ]; then
rm -f /bin/ovs-dpdkctl
fi
if [ -e "$CONFIG_FILE" ]; then
rm -f "$CONFIG_FILE"
fi
-}
+}
function useage {
cat << "EOF"