From f652e0fb6d46dd39cf9932234a04571c7ede3110 Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Tue, 21 May 2013 18:12:48 +0100 Subject: [PATCH] xenapi - quantum: L2 agents for dom0 and domU This patch makes it easier to support the all-in-one installation of XenServer with Quantum: - separate agents for domU and dom0 openvswitches. The domU agent's name is "q-domua", and the dom0 agent is called "q-agt" - create a physnet1 physical network to connect those openvswitches Note: This patch depends on the quantum patch: https://review.openstack.org/15022 For a diagram, see: http://goo.gl/BuAdg Change-Id: Iff851438553528b9051edd380ecbb2c0415cbf78 --- lib/nova | 5 +++- lib/quantum | 5 ++++ lib/quantum_plugins/openvswitch | 48 +++++++++++++++++---------------- tools/xen/functions | 26 ++++++++++++++++++ tools/xen/install_os_domU.sh | 21 +++++++++++++-- tools/xen/xenrc | 1 + 6 files changed, 80 insertions(+), 26 deletions(-) diff --git a/lib/nova b/lib/nova index 9fc4ded7b6..ea932a31d9 100644 --- a/lib/nova +++ b/lib/nova @@ -80,7 +80,10 @@ if [ "$VIRT_DRIVER" = 'xenserver' ]; then PUBLIC_INTERFACE_DEFAULT=eth3 GUEST_INTERFACE_DEFAULT=eth1 # Allow ``build_domU.sh`` to specify the flat network bridge via kernel args - FLAT_NETWORK_BRIDGE_DEFAULT=$(grep -o 'flat_network_bridge=[[:alnum:]]*' /proc/cmdline | cut -d= -f 2 | sort -u) + FLAT_NETWORK_BRIDGE_DEFAULT=$(sed -e 's/.* flat_network_bridge=\([[:alnum:]]*\).*$/\1/g' /proc/cmdline) + if is_service_enabled quantum; then + XEN_INTEGRATION_BRIDGE=$(sed -e 's/.* xen_integration_bridge=\([[:alnum:]]*\).*$/\1/g' /proc/cmdline) + fi elif [ "$VIRT_DRIVER" = 'baremetal' ]; then NETWORK_MANAGER=${NETWORK_MANAGER:-FlatManager} PUBLIC_INTERFACE_DEFAULT=eth0 diff --git a/lib/quantum b/lib/quantum index c36a743518..d85c6483aa 100644 --- a/lib/quantum +++ b/lib/quantum @@ -386,6 +386,11 @@ function start_quantum_agents() { screen_it q-l3 "cd $QUANTUM_DIR && python $AGENT_L3_BINARY --config-file $QUANTUM_CONF --config-file=$Q_L3_CONF_FILE" screen_it q-meta "cd $QUANTUM_DIR && python $AGENT_META_BINARY --config-file $QUANTUM_CONF --config-file=$Q_META_CONF_FILE" + if [ "$VIRT_DRIVER" = 'xenserver' ]; then + # For XenServer, start an agent for the domU openvswitch + screen_it q-domua "cd $QUANTUM_DIR && python $AGENT_BINARY --config-file $QUANTUM_CONF --config-file /$Q_PLUGIN_CONF_FILE.domU" + fi + if is_service_enabled q-lbaas; then screen_it q-lbaas "cd $QUANTUM_DIR && python $AGENT_LBAAS_BINARY --config-file $QUANTUM_CONF --config-file=$LBAAS_AGENT_CONF_FILENAME" fi diff --git a/lib/quantum_plugins/openvswitch b/lib/quantum_plugins/openvswitch index 374172014b..7b9835df9f 100644 --- a/lib/quantum_plugins/openvswitch +++ b/lib/quantum_plugins/openvswitch @@ -11,7 +11,9 @@ function quantum_plugin_create_nova_conf() { _quantum_ovs_base_configure_nova_vif_driver if [ "$VIRT_DRIVER" = 'xenserver' ]; then iniset $NOVA_CONF DEFAULT xenapi_vif_driver nova.virt.xenapi.vif.XenAPIOpenVswitchDriver - iniset $NOVA_CONF DEFAULT xenapi_ovs_integration_bridge $FLAT_NETWORK_BRIDGE + iniset $NOVA_CONF DEFAULT xenapi_ovs_integration_bridge $XEN_INTEGRATION_BRIDGE + # Disable nova's firewall so that it does not conflict with quantum + iniset $NOVA_CONF DEFAULT firewall_driver nova.virt.firewall.NoopFirewallDriver fi } @@ -71,6 +73,10 @@ function quantum_plugin_configure_plugin_agent() { AGENT_BINARY="$QUANTUM_DIR/bin/quantum-openvswitch-agent" if [ "$VIRT_DRIVER" = 'xenserver' ]; then + # Make a copy of our config for domU + sudo cp /$Q_PLUGIN_CONF_FILE "/$Q_PLUGIN_CONF_FILE.domu" + + # Deal with Dom0's L2 Agent: Q_RR_DOM0_COMMAND="$QUANTUM_DIR/bin/quantum-rootwrap-xen-dom0 $Q_RR_CONF_FILE" # For now, duplicate the xen configuration already found in nova.conf @@ -83,29 +89,25 @@ function quantum_plugin_configure_plugin_agent() { # that executes commands on dom0 via a XenAPI plugin. iniset /$Q_PLUGIN_CONF_FILE AGENT root_helper "$Q_RR_DOM0_COMMAND" - # FLAT_NETWORK_BRIDGE is the dom0 integration bridge. To - # ensure the bridge lacks direct connectivity, set - # VM_VLAN=-1;VM_DEV=invalid in localrc - iniset /$Q_PLUGIN_CONF_FILE OVS integration_bridge $FLAT_NETWORK_BRIDGE + # Set "physical" mapping + iniset /$Q_PLUGIN_CONF_FILE OVS bridge_mappings "physnet1:$FLAT_NETWORK_BRIDGE" - # The ovs agent needs to ensure that the ports associated with - # a given network share the same local vlan tag. On - # single-node XS/XCP, this requires monitoring both the dom0 - # bridge, where VM's are attached, and the domU bridge, where - # dhcp servers are attached. - if is_service_enabled q-dhcp; then - iniset /$Q_PLUGIN_CONF_FILE OVS domu_integration_bridge $OVS_BRIDGE - # DomU will use the regular rootwrap - iniset /$Q_PLUGIN_CONF_FILE AGENT domu_root_helper "$Q_RR_COMMAND" - # Plug the vm interface into the domU integration bridge. - sudo ip addr flush dev $GUEST_INTERFACE_DEFAULT - sudo ip link set $OVS_BRIDGE up - # Assign the VM IP only if it has been set explicitly - if [[ "$VM_IP" != "" ]]; then - sudo ip addr add $VM_IP dev $OVS_BRIDGE - fi - sudo ovs-vsctl add-port $OVS_BRIDGE $GUEST_INTERFACE_DEFAULT - fi + # XEN_INTEGRATION_BRIDGE is the integration bridge in dom0 + iniset /$Q_PLUGIN_CONF_FILE OVS integration_bridge $XEN_INTEGRATION_BRIDGE + + # Set up domU's L2 agent: + + # Create a bridge "br-$GUEST_INTERFACE_DEFAULT" + sudo ovs-vsctl --no-wait -- --may-exist add-br "br-$GUEST_INTERFACE_DEFAULT" + # Add $GUEST_INTERFACE_DEFAULT to that bridge + sudo ovs-vsctl add-port "br-$GUEST_INTERFACE_DEFAULT" $GUEST_INTERFACE_DEFAULT + + # Set bridge mappings to "physnet1:br-$GUEST_INTERFACE_DEFAULT" + iniset "/$Q_PLUGIN_CONF_FILE.domU" OVS bridge_mappings "physnet1:br-$GUEST_INTERFACE_DEFAULT" + # Set integration bridge to domU's + iniset "/$Q_PLUGIN_CONF_FILE.domU" OVS integration_bridge $OVS_BRIDGE + # Set root wrap + iniset "/$Q_PLUGIN_CONF_FILE.domU" AGENT root_helper "$Q_RR_COMMAND" fi } diff --git a/tools/xen/functions b/tools/xen/functions index c6e484d03e..3458263206 100644 --- a/tools/xen/functions +++ b/tools/xen/functions @@ -123,6 +123,32 @@ function _bridge_exists() { ! [ -z $(xe network-list bridge="$bridge" --minimal) ] } +function _network_uuid() { + local bridge_or_net_name + bridge_or_net_name=$1 + + if _bridge_exists "$bridge_or_net_name"; then + xe network-list bridge="$bridge_or_net_name" --minimal + else + xe network-list name-label="$bridge_or_net_name" --minimal + fi +} + +function add_interface() { + local vm_name + local bridge_or_network_name + + vm_name="$1" + bridge_or_network_name="$2" + device_number="$3" + + local vm + local net + + vm=$(xe vm-list name-label="$vm_name" --minimal) + net=$(_network_uuid "$bridge_or_network_name") + xe vif-create network-uuid=$net vm-uuid=$vm device=$device_number +} function setup_network() { local bridge_or_net_name diff --git a/tools/xen/install_os_domU.sh b/tools/xen/install_os_domU.sh index 161d7e774d..8b2a687119 100755 --- a/tools/xen/install_os_domU.sh +++ b/tools/xen/install_os_domU.sh @@ -71,6 +71,12 @@ setup_network "$VM_BRIDGE_OR_NET_NAME" setup_network "$MGT_BRIDGE_OR_NET_NAME" setup_network "$PUB_BRIDGE_OR_NET_NAME" +# With quantum, one more network is required, which is internal to the +# hypervisor, and used by the VMs +if is_service_enabled quantum; then + setup_network "$XEN_INT_BRIDGE_OR_NET_NAME" +fi + if parameter_is_specified "FLAT_NETWORK_BRIDGE"; then cat >&2 << EOF ERROR: FLAT_NETWORK_BRIDGE is specified in localrc file @@ -195,6 +201,12 @@ if [ -z "$templateuuid" ]; then # create a new VM with the given template # creating the correct VIFs and metadata FLAT_NETWORK_BRIDGE=$(bridge_for "$VM_BRIDGE_OR_NET_NAME") + + KERNEL_PARAMS_FOR_QUANTUM="" + if is_service_enabled quantum; then + XEN_INTEGRATION_BRIDGE=$(bridge_for "$XEN_INT_BRIDGE_OR_NET_NAME") + KERNEL_PARAMS_FOR_QUANTUM="xen_integration_bridge=${XEN_INTEGRATION_BRIDGE}" + fi $THIS_DIR/scripts/install-os-vpx.sh \ -t "$UBUNTU_INST_TEMPLATE_NAME" \ -v "$VM_BRIDGE_OR_NET_NAME" \ @@ -202,7 +214,7 @@ if [ -z "$templateuuid" ]; then -p "$PUB_BRIDGE_OR_NET_NAME" \ -l "$GUEST_NAME" \ -r "$OSDOMU_MEM_MB" \ - -k "flat_network_bridge=${FLAT_NETWORK_BRIDGE}" + -k "flat_network_bridge=${FLAT_NETWORK_BRIDGE} ${KERNEL_PARAMS_FOR_QUANTUM}" # wait for install to finish wait_for_VM_to_halt @@ -240,11 +252,16 @@ fi # $THIS_DIR/build_xva.sh "$GUEST_NAME" +# Attach a network interface for the integration network (so that the bridge +# is created by XenServer). This is required for Quantum. +if is_service_enabled quantum; then + add_interface "$GUEST_NAME" "$XEN_INT_BRIDGE_OR_NET_NAME" "4" +fi + # create a snapshot before the first boot # to allow a quick re-run with the same settings xe vm-snapshot vm="$GUEST_NAME" new-name-label="$SNAME_FIRST_BOOT" - # # Run DevStack VM # diff --git a/tools/xen/xenrc b/tools/xen/xenrc index 7aaafd219d..0ed3a6a7e5 100644 --- a/tools/xen/xenrc +++ b/tools/xen/xenrc @@ -25,6 +25,7 @@ OSDOMU_VDI_GB=8 MGT_BRIDGE_OR_NET_NAME="xenbr0" VM_BRIDGE_OR_NET_NAME="OpenStack VM Network" PUB_BRIDGE_OR_NET_NAME="OpenStack Public Network" +XEN_INT_BRIDGE_OR_NET_NAME="OpenStack VM Integration Network" # VM Password GUEST_PASSWORD=${GUEST_PASSWORD:-secrete}