adds ovs-dpdkctl tool
- This change introduces a new tool for configuring host interfaces for use with ovs and dpdk. - The ovs-dpdkctl tool will be executed via systemd when a system first boots to bind interfaces to dpdk compaible drivers. - The ovs-dpdkctl tool will be injected into the ovsdb container to allow external configuration of ovs bridges and ports for use with dpdk. Change-Id: Ie8f32d097f0a6816c2ddd03ade926c00837da322
This commit is contained in:
Sean Mooney
parent
4b5b14948b
commit
c29244a380
390
tools/ovs-dpdkctl.sh
Executable file
390
tools/ovs-dpdkctl.sh
Executable file
@ -0,0 +1,390 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
_XTRACE_OVS_DPDK_CTL=$(set +o | grep xtrace)
|
||||||
|
if [[ "${OVS_DPDK_CTL_DEBUG}" == "True" ]]; then
|
||||||
|
set -o xtrace
|
||||||
|
fi
|
||||||
|
|
||||||
|
FULL_PATH=$(realpath "${BASH_SOURCE[0]}")
|
||||||
|
CONFIG_FILE=${CONFIG_FILE:-"/etc/default/ovs-dpdk.conf"}
|
||||||
|
SERVICE_FILE="/etc/systemd/system/ovs-dpdkctl.service"
|
||||||
|
|
||||||
|
function get_value {
|
||||||
|
crudini --get $CONFIG_FILE $@
|
||||||
|
}
|
||||||
|
|
||||||
|
function set_value {
|
||||||
|
crudini --set $CONFIG_FILE $1 $2 "$3"
|
||||||
|
}
|
||||||
|
|
||||||
|
function del_value {
|
||||||
|
crudini --del $CONFIG_FILE $@
|
||||||
|
}
|
||||||
|
|
||||||
|
function is_set {
|
||||||
|
output=$(crudini --get $CONFIG_FILE $* &> /dev/null ) && [ -n "$(crudini --get $CONFIG_FILE $*)" ]; echo $?
|
||||||
|
}
|
||||||
|
|
||||||
|
function show_config {
|
||||||
|
cat $CONFIG_FILE
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_config {
|
||||||
|
echo $CONFIG_FILE
|
||||||
|
}
|
||||||
|
|
||||||
|
function del_config {
|
||||||
|
rm -f $CONFIG_FILE
|
||||||
|
}
|
||||||
|
|
||||||
|
function generate_pciwhitelist {
|
||||||
|
local _Whitelist=''
|
||||||
|
for nic in $(list_dpdk_nics); do
|
||||||
|
address="$(get_value $nic address)"
|
||||||
|
if [ "$_Whitelist" == '' ]; then
|
||||||
|
_Whitelist="-w $address"
|
||||||
|
else
|
||||||
|
_Whitelist="$_Whitelist -w $address"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo $_Whitelist
|
||||||
|
}
|
||||||
|
|
||||||
|
function gen_port_mappings {
|
||||||
|
OVS_BRIDGE_MAPPINGS=$(get_value ovs bridge_mappings)
|
||||||
|
OVS_BRIDGES=${OVS_BRIDGE_MAPPINGS//,/ }
|
||||||
|
OVS_DPDK_PORT_MAPPINGS=""
|
||||||
|
ARRAY=( $OVS_BRIDGES )
|
||||||
|
for net in "${ARRAY[@]}"; do
|
||||||
|
bridge="${net##*:}"
|
||||||
|
nic=${bridge/br-/}
|
||||||
|
if [[ -z "$OVS_DPDK_PORT_MAPPINGS" ]]; then
|
||||||
|
OVS_DPDK_PORT_MAPPINGS="$nic:$bridge"
|
||||||
|
else
|
||||||
|
OVS_DPDK_PORT_MAPPINGS="$OVS_DPDK_PORT_MAPPINGS,$nic:$bridge"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "$OVS_DPDK_PORT_MAPPINGS"
|
||||||
|
}
|
||||||
|
|
||||||
|
function gen_config {
|
||||||
|
del_config
|
||||||
|
touch $CONFIG_FILE
|
||||||
|
set_value ovs bridge_mappings ${bridge_mappings:-""}
|
||||||
|
set_value ovs port_mappings ${port_mappings:-$(gen_port_mappings)}
|
||||||
|
set_value ovs cidr_mappings ${cidr_mappings:-""}
|
||||||
|
set_value ovs ovs_coremask ${ovs_coremask:-"0x1"}
|
||||||
|
set_value ovs pmd_coremask ${pmd_coremask:-"0x2"}
|
||||||
|
set_value ovs ovs_mem_channels ${ovs_mem_channels:-4}
|
||||||
|
set_value ovs ovs_socket_mem ${ovs_socket_mem:-"512"}
|
||||||
|
set_value ovs dpdk_interface_driver ${dpdk_interface_driver:-"uio_pci_generic"}
|
||||||
|
set_value ovs hugepage_mountpoint ${hugepage_mountpoint:-"/dev/hugepages"}
|
||||||
|
|
||||||
|
ls -al /sys/class/net/* | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print "\n[" a[n] "]\naddress = " a[n-2] "\ndriver ="}' >> $CONFIG_FILE
|
||||||
|
|
||||||
|
for nic in $(get_value | grep -v ovs); do
|
||||||
|
set_value $nic driver $(get_driver_by_address $(get_value $nic address))
|
||||||
|
done
|
||||||
|
for nic in $(list_dpdk_nics); do
|
||||||
|
set_value $nic driver ${dpdk_interface_driver:-"uio_pci_generic"}
|
||||||
|
done
|
||||||
|
set_value ovs pci_whitelist "'${pci_whitelist:-$(generate_pciwhitelist)}'"
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
function bind_nic {
|
||||||
|
echo $1 > /sys/bus/pci/drivers/$2/bind
|
||||||
|
echo $2 > /sys/bus/pci/devices/$1/driver_override
|
||||||
|
}
|
||||||
|
|
||||||
|
function unbind_nic {
|
||||||
|
echo $1 > /sys/bus/pci/drivers/$2/unbind
|
||||||
|
echo > /sys/bus/pci/devices/$1/driver_override
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
function list_dpdk_nics {
|
||||||
|
for nic in $(crudini --get $CONFIG_FILE ovs port_mappings | cut -d : -f 1); do
|
||||||
|
echo $nic;
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function bind_nics {
|
||||||
|
for nic in $(list_dpdk_nics); do
|
||||||
|
device_address="$(get_value $nic address)"
|
||||||
|
current_driver="$(get_driver_by_address $device_address)"
|
||||||
|
target_driver="$(get_value $nic driver)"
|
||||||
|
if [ "$current_driver" != "$target_driver" ]; then
|
||||||
|
set_value $nic old_driver $current_driver
|
||||||
|
unbind_nic $device_address $current_driver
|
||||||
|
bind_nic $device_address $target_driver
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function unbind_nics {
|
||||||
|
for nic in $(list_dpdk_nics); do
|
||||||
|
if [ "$(is_set $nic old_driver)" == 0 ]; then
|
||||||
|
device_address="$(get_value $nic address)"
|
||||||
|
current_driver="$(get_driver_by_address $device_address)"
|
||||||
|
target_driver="$(get_value $nic old_driver)"
|
||||||
|
if [ "$current_driver" != "$target_driver" ]; then
|
||||||
|
unbind_nic $device_address $current_driver
|
||||||
|
bind_nic $device_address $target_driver
|
||||||
|
del_value $nic old_driver
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function get_address_by_name {
|
||||||
|
ls -al /sys/class/net/$1 | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print a[n-2] }'
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_driver_by_address {
|
||||||
|
ls /sys/bus/pci/devices/$1/driver -al | awk '{n=split($NF,a,"/"); print a[n]}'
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_port_bridge {
|
||||||
|
for pair in $(get_value ovs port port_mappings); do
|
||||||
|
nic=`echo $pair | cut -f 1 -d ":"`
|
||||||
|
if [[ "$nic" == "$1" ]]; then
|
||||||
|
bridge=`echo $pair | cut -f 2 -d ":"`
|
||||||
|
echo $bridge
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
function init_ovs_db {
|
||||||
|
ovs-vsctl init
|
||||||
|
ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask="$(get_value ovs pmd_coremask)" \
|
||||||
|
other_config:dpdk-init=True other_config:dpdk-lcore-mask="$(get_value ovs ovs_coremask)" \
|
||||||
|
other_config:dpdk-mem-channels="$(get_value ovs ovs_mem_channels)" \
|
||||||
|
other_config:dpdk-socket-mem="$(get_value ovs ovs_socket_mem)" \
|
||||||
|
other_config:dpdk-hugepage-dir="$(get_value ovs hugepage_mountpoint)" \
|
||||||
|
other_config:dpdk-extra=" --proc-type primary $(get_value ovs pci_whitelist) "
|
||||||
|
}
|
||||||
|
|
||||||
|
function init_ovs_bridges {
|
||||||
|
raw_bridge_mappings=$(get_value ovs bridge_mappings)
|
||||||
|
bridge_mappings=( ${raw_bridge_mappings//,/ } )
|
||||||
|
for pair in "${bridge_mappings[@]}"; do
|
||||||
|
bridge=`echo $pair | cut -f 2 -d ":"`
|
||||||
|
sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge -- set Bridge $bridge datapath_type=netdev
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function init_ovs_interfaces {
|
||||||
|
local pci_port_pairs ==''
|
||||||
|
for nic in $(list_dpdk_nics); do
|
||||||
|
address="$(get_value $nic address)"
|
||||||
|
if [ "$pci_port_pairs" == '' ]; then
|
||||||
|
pci_port_pairs ="$address,$nic"
|
||||||
|
else
|
||||||
|
pci_port_pairs="$pci_port_pairs $address,$nic"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
pci_port_pairs="$(echo $pci_port_pairs | sort)"
|
||||||
|
dpdk_port_number=0
|
||||||
|
for pair in $pci_port_pairs; do
|
||||||
|
addr="$(echo $pair | cut -f 1 -d ",")"
|
||||||
|
nic="$(echo $pair | cut -f 2 -d ",")"
|
||||||
|
bridge="$(get_port_bridge $nic)"
|
||||||
|
# ovs 2.6 and older requires dpdkX names, ovs 2.7+ requires dpdk-devargs instead.
|
||||||
|
ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \
|
||||||
|
-- set Interface "dpdk${dpdk_port_number}" type=dpdk || \
|
||||||
|
ovs-vsctl --no-wait --may-exist add-port $bridge $nic \
|
||||||
|
-- set Interface $nic type=dpdk options:dpdk-devargs=$addr
|
||||||
|
|
||||||
|
dpdk_port_number=$((dpdk_port_number+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
function init {
|
||||||
|
init_ovs_db
|
||||||
|
init_ovs_bridges
|
||||||
|
init_ovs_interfaces
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_service {
|
||||||
|
cat << EOF | tee "$SERVICE_FILE"
|
||||||
|
|
||||||
|
[Unit]
|
||||||
|
Description=configuration service for ovs-dpdk nics.
|
||||||
|
Before=network-pre.target
|
||||||
|
After=syslog.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
# Uncomment to enable debug logging.
|
||||||
|
# Environment=OVS_DPDK_CTL_DEBUG=True
|
||||||
|
Environment=CONFIG_FILE=$CONFIG_FILE
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=/bin/ovs-dpdkctl bind_nics
|
||||||
|
ExecStop=/bin/ovs-dpdkctl unbind_nics
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
EOF
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable ovs-dpdkctl
|
||||||
|
}
|
||||||
|
|
||||||
|
function uninstall_service {
|
||||||
|
systemctl disable ovs-dpdkctl
|
||||||
|
rm -f "$SERVICE_FILE"
|
||||||
|
systemctl daemon-reload
|
||||||
|
}
|
||||||
|
|
||||||
|
function install {
|
||||||
|
if [ ! -e "$SERVICE_FILE" ]; then
|
||||||
|
install_service
|
||||||
|
fi
|
||||||
|
if [ ! -e /bin/ovs-dpdkctl ]; then
|
||||||
|
cp "$FULL_PATH" /bin/ovs-dpdkctl
|
||||||
|
chmod +x /bin/ovs-dpdkctl
|
||||||
|
fi
|
||||||
|
if [ ! -e "$CONFIG_FILE" ]; then
|
||||||
|
gen_config
|
||||||
|
fi
|
||||||
|
systemctl start ovs-dpdkctl
|
||||||
|
}
|
||||||
|
|
||||||
|
function uninstall {
|
||||||
|
systemctl stop ovs-dpdkctl
|
||||||
|
if [ -e "$SERVICE_FILE" ]; then
|
||||||
|
uninstall_service
|
||||||
|
fi
|
||||||
|
if [ -e /bin/ovs-dpdkctl ]; then
|
||||||
|
rm -f /bin/ovs-dpdkctl
|
||||||
|
fi
|
||||||
|
if [ -e "$CONFIG_FILE" ]; then
|
||||||
|
rm -f "$CONFIG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function useage {
|
||||||
|
cat << "EOF"
|
||||||
|
ovs-dpdkctl.sh: A tool to configure ovs with dpdk.
|
||||||
|
|
||||||
|
- This tool automate the process of binding host insterfacesto a dpdk
|
||||||
|
compaible driver (uio_pci_generic | vfio-pci) at boot.
|
||||||
|
- This tool automate bootstraping ovs so that it can use the
|
||||||
|
dpdk accelerated netdev datapath.
|
||||||
|
|
||||||
|
commands:
|
||||||
|
- install:
|
||||||
|
- installs ovs-dpdkctl as a systemd service.
|
||||||
|
- installs ovs-dpdkctl binary.
|
||||||
|
- generates ovs-dpdkctl configuration file.
|
||||||
|
- starts ovs-dpdkctl service.
|
||||||
|
- uninstall:
|
||||||
|
- stops ovs-dpdkctl service.
|
||||||
|
- uninstalls ovs-dpdkctl systemd service.
|
||||||
|
- uninstalls ovs-dpdkctl binary.
|
||||||
|
- removes ovs-dpdkctl configuration file.
|
||||||
|
- bind_nics:
|
||||||
|
- iterates over all dpdk interfaces defined in ovs-dpdkctl config
|
||||||
|
and binds the interface to the target driver specifed in the config
|
||||||
|
if current driver does not equal target.
|
||||||
|
- unbind_nics:
|
||||||
|
- iterates over all dpdk interfaces defined in ovs-dpdkctl config
|
||||||
|
and restores the interface to its original non dpdk driver.
|
||||||
|
- init:
|
||||||
|
- defines dpdk specific configuration paramater in the ovsdb.
|
||||||
|
- creates bridges as spcified by ovs bridge_mappings in
|
||||||
|
ovs-dpdkctl config.
|
||||||
|
- creates dpdk ports as defined by ovs port_mappings in
|
||||||
|
ovs-dpdkctl config.
|
||||||
|
- useage:
|
||||||
|
- prints this message
|
||||||
|
|
||||||
|
options:
|
||||||
|
- debuging:
|
||||||
|
- To enable debuging export OVS_DPDK_CTL_DEBUG=True
|
||||||
|
- install:
|
||||||
|
- The varibles discribed below can be defined to customise
|
||||||
|
installation of ovs-dpdkctl.
|
||||||
|
<variable>=<value> ovs-dpdkctl.sh install
|
||||||
|
- bridge_mappings:
|
||||||
|
- A comma seperated list of physnet to bridge mappings.
|
||||||
|
- Example: bridge_mappings=physnet1:br-ex1,physnet2:br-ex2
|
||||||
|
- Default: ""
|
||||||
|
- port_mappings:
|
||||||
|
- A comma seperated list of port to bridge mappings.
|
||||||
|
- Example: port_mappings=eth1:br-ex1,eth2:br-ex2
|
||||||
|
- Default: generated form bridge_mappings assuming bridge names
|
||||||
|
are constructed by appending br- to port name.
|
||||||
|
- cidr_mappings:
|
||||||
|
- A comma seperated list of bridge to cidr mappings.
|
||||||
|
- Example: cidr_mappings=br-ex1:192.168.1.1/24,br-ex2:192.168.2.1/24
|
||||||
|
- Default: ""
|
||||||
|
- ovs_coremask:
|
||||||
|
- A hex encoded string container a bitmask of what cpu core
|
||||||
|
to pin the non dataplane treads of the ovs-vswitchd to.
|
||||||
|
- Node that only the first core of the bit mask is currently
|
||||||
|
used by ovs.
|
||||||
|
- Example: ovs_coremask=0x1
|
||||||
|
- Default: "0x1"
|
||||||
|
- pmd_coremask:
|
||||||
|
- A hex encoded string container a bitmask of what cpu cores
|
||||||
|
to pin the dataplane pool mode driver treads of the ovs-vswitchd to.
|
||||||
|
- Each bit set in the bitmask will result in the creating of a pmd.
|
||||||
|
- For best performance it is recomended to allocate at least 1 pmd per
|
||||||
|
numa node. On systems with HyperThreading enabled it is recomended to also
|
||||||
|
allocate the HT sibling core in the pmd_coremask.cores allocated
|
||||||
|
to ovs with dpdk via the pmd_coremask should be removed from the
|
||||||
|
nova vcpu_pin_set and isolated from the kernel scheduler.
|
||||||
|
- Note it is not recommended to isolate cores in the nova vcpu_pin_set
|
||||||
|
unless the host will be dedicated for vms that request cpu pinning.
|
||||||
|
- Example: pmd_coremask=0x4
|
||||||
|
- Default: "0x4"
|
||||||
|
- ovs_mem_channels:
|
||||||
|
- The number of memory channels supported by the plathforms.
|
||||||
|
- Example: ovs_mem_channels=2
|
||||||
|
- Default: "4"
|
||||||
|
- ovs_socket_mem:
|
||||||
|
- A comma seperated list of hugepage memory, specifed in MBs per numa node,
|
||||||
|
allocated to the ovs-vswitchd to use for the dpdk dataplane.
|
||||||
|
- For best performance memory should be allocated evenly across all numa node
|
||||||
|
that will run a pmd.
|
||||||
|
- Example: ovs_socket_mem=512,512
|
||||||
|
- Default: "512"
|
||||||
|
- hugepage_mountpoint:
|
||||||
|
- The hugepage mountpoint to use when allocating memory for dpdk.
|
||||||
|
- Example: hugepage_mountpoint=/dev/my_custom_mountpoint
|
||||||
|
- Default: "/dev/hugepages"
|
||||||
|
- dpdk_interface_driver:
|
||||||
|
- The dpdk compatible userspace driver to use when binding host interfaces.
|
||||||
|
- Example: dpdk_interface_driver=vfio_pci
|
||||||
|
- Default: "uio_pci_generic"
|
||||||
|
- pci_whitelist:
|
||||||
|
- A repeated space seperated list of pci whitelist flags
|
||||||
|
for allowed ovs-dpdk ports.
|
||||||
|
- The pci_whitelist allows multiple dpdk primary process to
|
||||||
|
utilise different pci devices without resulting in a conflict
|
||||||
|
of ownership.
|
||||||
|
- Example: pci_whitelist="-w <pci address 1> -w <pci address 2>"
|
||||||
|
- Default: auto generated form port_mappings.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ $# -ge 1 ]; then
|
||||||
|
func=$1
|
||||||
|
shift
|
||||||
|
else
|
||||||
|
func="useage"
|
||||||
|
fi
|
||||||
|
|
||||||
|
#replace with switch later
|
||||||
|
eval "$func $@"
|
||||||
|
|
||||||
|
|
||||||
|
${_XTRACE_OVS_DPDK_CTL}
|
||||||
Loading…
Reference in New Issue
Block a user