Merge changes Ie7cb4858,Iadb6e181,I2dc31acb,Ia3e2e65b

* changes: Remove cloning of devstack Multiple vpx for xen, post splitting of script Add logging for prepare_guest Generalize xen network config
2012-02-17 21:44:11 +00:00 · 2012-02-17 21:44:11 +00:00 · 45f5f4178e
commit 45f5f4178e
parent 6a3f607272 0d44b8bd63
10 changed files with 189 additions and 68 deletions
--- a/stack.sh
+++ b/stack.sh
@ -200,14 +200,13 @@ LIBVIRT_TYPE=${LIBVIRT_TYPE:-kvm}
 # cases unless you are working on multi-zone mode.
 SCHEDULER=${SCHEDULER:-nova.scheduler.simple.SimpleScheduler}

+HOST_IP_IFACE=${HOST_IP_IFACE:-eth0}
 # Use the eth0 IP unless an explicit is set by ``HOST_IP`` environment variable
-if [ ! -n "$HOST_IP" ]; then
-    HOST_IP=`LC_ALL=C /sbin/ifconfig eth0 | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'`
+if [ -z "$HOST_IP" -o "$HOST_IP" == "dhcp" ]; then
+    HOST_IP=`LC_ALL=C /sbin/ifconfig ${HOST_IP_IFACE} | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'`
    if [ "$HOST_IP" = "" ]; then
        echo "Could not determine host ip address."
-        echo "If this is not your first run of stack.sh, it is "
-        echo "possible that nova moved your eth0 ip address to the FLAT_NETWORK_BRIDGE."
-        echo "Please specify your HOST_IP in your localrc."
+        echo "Either localrc specified dhcp on ${HOST_IP_IFACE} or defaulted to eth0"
        exit 1
    fi
 fi
--- a/tools/xen/build_domU.sh
+++ b/tools/xen/build_domU.sh
@ -37,51 +37,85 @@ if ! which git; then
 fi

 # Helper to create networks
+# Uses echo trickery to return network uuid
 function create_network() {
-    if ! xe network-list | grep bridge | grep -q $1; then
-        echo "Creating bridge $1"
-        xe network-create name-label=$1
+    br=$1
+    dev=$2
+    vlan=$3
+    netname=$4
+    if [ -z $br ]
+    then
+        pif=$(xe pif-list --minimal device=$dev VLAN=$vlan)
+        if [ -z $pif ]
+        then
+            net=$(xe network-create name-label=$netname)
+        else
+            net=$(xe network-list --minimal PIF-uuids=$pif)
+        fi
+        echo $net
+        return 0
+    fi
+    if [ ! $(xe network-list --minimal params=bridge | grep -w --only-matching $br) ]
+    then
+        echo "Specified bridge $br does not exist"
+        echo "If you wish to use defaults, please keep the bridge name empty"
+        exit 1
+    else
+        net=$(xe network-list --minimal bridge=$br)
+        echo $net
+    fi
+}
+
+function errorcheck() {
+    rc=$?
+    if [ $rc -ne 0 ]
+    then
+        exit $rc
    fi
 }

 # Create host, vm, mgmt, pub networks
-create_network xapi0
-create_network $VM_BR
-create_network $MGT_BR
-create_network $PUB_BR
-
-# Get the uuid for our physical (public) interface
-PIF=`xe pif-list --minimal device=eth0`
-
-# Create networks/bridges for vm and management
-VM_NET=`xe network-list --minimal bridge=$VM_BR`
-MGT_NET=`xe network-list --minimal bridge=$MGT_BR`
+VM_NET=$(create_network "$VM_BR" "$VM_DEV" "$VM_VLAN" "vmbr")
+errorcheck
+MGT_NET=$(create_network "$MGT_BR" "$MGT_DEV" "$MGT_VLAN" "mgtbr")
+errorcheck
+PUB_NET=$(create_network "$PUB_BR" "$PUB_DEV" "$PUB_VLAN" "pubbr")
+errorcheck

 # Helper to create vlans
 function create_vlan() {
-    pif=$1
+    dev=$1
    vlan=$2
    net=$3
-    if ! xe vlan-list | grep tag | grep -q $vlan; then
-        xe vlan-create pif-uuid=$pif vlan=$vlan network-uuid=$net
+    # VLAN -1 refers to no VLAN (physical network)
+    if [ $vlan -eq -1 ]
+    then
+        return
+    fi
+    if [ -z $(xe vlan-list --minimal tag=$vlan) ]
+    then
+        pif=$(xe pif-list --minimal network-uuid=$net)
+        # We created a brand new network this time
+        if [ -z $pif ]
+        then
+            pif=$(xe pif-list --minimal device=$dev VLAN=-1)
+            xe vlan-create pif-uuid=$pif vlan=$vlan network-uuid=$net
+        else
+            echo "VLAN does not exist but PIF attached to this network"
+            echo "How did we reach here?"
+            exit 1
+        fi
    fi
 }

 # Create vlans for vm and management
-create_vlan $PIF $VM_VLAN $VM_NET
-create_vlan $PIF $MGT_VLAN $MGT_NET
+create_vlan $PUB_DEV $PUB_VLAN $PUB_NET
+create_vlan $VM_DEV $VM_VLAN $VM_NET
+create_vlan $MGT_DEV $MGT_VLAN $MGT_NET

 # dom0 ip
 HOST_IP=${HOST_IP:-`ifconfig xenbr0 | grep "inet addr" | cut -d ":" -f2 | sed "s/ .*//"`}

-# Setup host-only nat rules
-HOST_NET=169.254.0.0/16
-if ! iptables -L -v -t nat | grep -q $HOST_NET; then
-    iptables -t nat -A POSTROUTING -s $HOST_NET -j SNAT --to-source $HOST_IP
-    iptables -I FORWARD 1 -s $HOST_NET -j ACCEPT
-    /etc/init.d/iptables save
-fi
-
 # Set up ip forwarding
 if ! grep -q "FORWARD_IPV4=YES" /etc/sysconfig/network; then
    # FIXME: This doesn't work on reboot!
@ -139,7 +173,16 @@ if [ "$DO_SHUTDOWN" = "1" ]; then
 fi

 # Start guest
-$TOP_DIR/scripts/install-os-vpx.sh -f $XVA -v $VM_BR -m $MGT_BR -p $PUB_BR
+if [ -z $VM_BR ]; then
+    VM_BR=$(xe network-list --minimal uuid=$VM_NET params=bridge)
+fi
+if [ -z $MGT_BR ]; then
+    MGT_BR=$(xe network-list --minimal uuid=$MGT_NET params=bridge)
+fi
+if [ -z $PUB_BR ]; then
+    PUB_BR=$(xe network-list --minimal uuid=$PUB_NET params=bridge)
+fi
+$TOP_DIR/scripts/install-os-vpx.sh -f $XVA -v $VM_BR -m $MGT_BR -p $PUB_BR -l $GUEST_NAME -w

 # If we have copied our ssh credentials, use ssh to monitor while the installation runs
 WAIT_TILL_LAUNCH=${WAIT_TILL_LAUNCH:-1}
--- a/tools/xen/build_domU_multi.sh
+++ b/tools/xen/build_domU_multi.sh
@ -17,19 +17,19 @@ FLOATING_RANGE=${FLOATING_RANGE:-192.168.1.196/30}
 COMMON_VARS="$STACKSH_PARAMS MYSQL_HOST=$HEAD_MGT_IP RABBIT_HOST=$HEAD_MGT_IP GLANCE_HOSTPORT=$HEAD_MGT_IP:9292 FLOATING_RANGE=$FLOATING_RANGE"

 # Helper to launch containers
-function build_domU {
-    GUEST_NAME=$1 PUB_IP=$2 MGT_IP=$3 DO_SHUTDOWN=$4 TERMINATE=$TERMINATE STACKSH_PARAMS="$COMMON_VARS $5" ./build_domU.sh
+function build_xva {
+    GUEST_NAME=$1 PUB_IP=$2 MGT_IP=$3 DO_SHUTDOWN=$4 TERMINATE=$TERMINATE STACKSH_PARAMS="$COMMON_VARS $5" ./build_xva.sh
 }

 # Launch the head node - headnode uses a non-ip domain name,
 # because rabbit won't launch with an ip addr hostname :(
-build_domU HEADNODE $HEAD_PUB_IP $HEAD_MGT_IP 1 "ENABLED_SERVICES=g-api,g-reg,key,n-api,n-sch,n-vnc,horizon,mysql,rabbit"
+build_xva HEADNODE $HEAD_PUB_IP $HEAD_MGT_IP 1 "ENABLED_SERVICES=g-api,g-reg,key,n-api,n-sch,n-vnc,horizon,mysql,rabbit"

 # Wait till the head node is up
-while ! curl -L http://$HEAD_PUB_IP | grep -q username; do
-    echo "Waiting for head node ($HEAD_PUB_IP) to start..."
-    sleep 5
-done
+#while ! curl -L http://$HEAD_PUB_IP | grep -q username; do
+#    echo "Waiting for head node ($HEAD_PUB_IP) to start..."
+#    sleep 5
+#done

 # Build the HA compute host
-build_domU $COMPUTE_PUB_IP $COMPUTE_PUB_IP $COMPUTE_MGT_IP 0 "ENABLED_SERVICES=n-cpu,n-net,n-api"
+build_xva COMPUTENODE $COMPUTE_PUB_IP $COMPUTE_MGT_IP 0 "ENABLED_SERVICES=n-cpu,n-net,n-api"
--- a/tools/xen/build_xva.sh
+++ b/tools/xen/build_xva.sh
@ -62,7 +62,7 @@ XVA_DIR=$TOP_DIR/xvas
 mkdir -p $XVA_DIR

 # Path to xva
-XVA=$XVA_DIR/$GUEST_NAME.xva 
+XVA=$XVA_DIR/$GUEST_NAME.xva

 # Setup fake grub
 rm -rf $STAGING_DIR/boot/grub/
@ -102,7 +102,9 @@ sed -e "s,@BUILD_NUMBER@,$BUILD_NUMBER,g" -i $OVA

 # Run devstack on launch
 cat <<EOF >$STAGING_DIR/etc/rc.local
-GUEST_PASSWORD=$GUEST_PASSWORD STAGING_DIR=/ DO_TGZ=0 bash /opt/stack/devstack/tools/xen/prepare_guest.sh
+# network restart required for getting the right gateway
+/etc/init.d/networking restart
+GUEST_PASSWORD=$GUEST_PASSWORD STAGING_DIR=/ DO_TGZ=0 bash /opt/stack/devstack/tools/xen/prepare_guest.sh > /opt/stack/prepare_guest.log 2>&1
 su -c "/opt/stack/run.sh > /opt/stack/run.sh.log" stack
 exit 0
 EOF
@ -122,12 +124,36 @@ EOF
 # Configure the network
 INTERFACES=$STAGING_DIR/etc/network/interfaces
 cp $TEMPLATES_DIR/interfaces.in  $INTERFACES
-sed -e "s,@ETH1_IP@,$VM_IP,g" -i $INTERFACES
-sed -e "s,@ETH1_NETMASK@,$VM_NETMASK,g" -i $INTERFACES
-sed -e "s,@ETH2_IP@,$MGT_IP,g" -i $INTERFACES
-sed -e "s,@ETH2_NETMASK@,$MGT_NETMASK,g" -i $INTERFACES
-sed -e "s,@ETH3_IP@,$PUB_IP,g" -i $INTERFACES
-sed -e "s,@ETH3_NETMASK@,$PUB_NETMASK,g" -i $INTERFACES
+if [ $VM_IP == "dhcp" ]; then
+    echo 'eth1 on dhcp'
+    sed -e "s,iface eth1 inet static,iface eth1 inet dhcp,g" -i $INTERFACES
+    sed -e '/@ETH1_/d' -i $INTERFACES
+else
+    sed -e "s,@ETH1_IP@,$VM_IP,g" -i $INTERFACES
+    sed -e "s,@ETH1_NETMASK@,$VM_NETMASK,g" -i $INTERFACES
+fi
+
+if [ $MGT_IP == "dhcp" ]; then
+    echo 'eth2 on dhcp'
+    sed -e "s,iface eth2 inet static,iface eth2 inet dhcp,g" -i $INTERFACES
+    sed -e '/@ETH2_/d' -i $INTERFACES
+else
+    sed -e "s,@ETH2_IP@,$MGT_IP,g" -i $INTERFACES
+    sed -e "s,@ETH2_NETMASK@,$MGT_NETMASK,g" -i $INTERFACES
+fi
+
+if [ $PUB_IP == "dhcp" ]; then
+    echo 'eth3 on dhcp'
+    sed -e "s,iface eth3 inet static,iface eth3 inet dhcp,g" -i $INTERFACES
+    sed -e '/@ETH3_/d' -i $INTERFACES
+else
+    sed -e "s,@ETH3_IP@,$PUB_IP,g" -i $INTERFACES
+    sed -e "s,@ETH3_NETMASK@,$PUB_NETMASK,g" -i $INTERFACES
+fi
+
+if [ -h $STAGING_DIR/sbin/dhclient3 ]; then
+    rm -f $STAGING_DIR/sbin/dhclient3
+fi

 # Gracefully cp only if source file/dir exists
 function cp_it {
@ -151,7 +177,7 @@ cat <<EOF >$STAGING_DIR/opt/stack/run.sh
 #!/bin/bash
 cd /opt/stack/devstack
 killall screen
-UPLOAD_LEGACY_TTY=yes HOST_IP=$PUB_IP VIRT_DRIVER=xenserver FORCE=yes MULTI_HOST=1 $STACKSH_PARAMS ./stack.sh
+UPLOAD_LEGACY_TTY=yes HOST_IP=$PUB_IP VIRT_DRIVER=xenserver FORCE=yes MULTI_HOST=1 HOST_IP_IFACE=$HOST_IP_IFACE $STACKSH_PARAMS ./stack.sh
 EOF
 chmod 755 $STAGING_DIR/opt/stack/run.sh

--- a/tools/xen/install_domU_multi.sh
+++ b/tools/xen/install_domU_multi.sh
@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Echo commands
+set -o xtrace
+
+# Head node host, which runs glance, api, keystone
+HEAD_PUB_IP=${HEAD_PUB_IP:-192.168.1.57}
+HEAD_MGT_IP=${HEAD_MGT_IP:-172.16.100.57}
+
+COMPUTE_PUB_IP=${COMPUTE_PUB_IP:-192.168.1.58}
+COMPUTE_MGT_IP=${COMPUTE_MGT_IP:-172.16.100.58}
+
+# Networking params
+FLOATING_RANGE=${FLOATING_RANGE:-192.168.1.196/30}
+
+# Variables common amongst all hosts in the cluster
+COMMON_VARS="$STACKSH_PARAMS MYSQL_HOST=$HEAD_MGT_IP RABBIT_HOST=$HEAD_MGT_IP GLANCE_HOSTPORT=$HEAD_MGT_IP:9292 FLOATING_RANGE=$FLOATING_RANGE"
+
+# Helper to launch containers
+function install_domU {
+    GUEST_NAME=$1 PUB_IP=$2 MGT_IP=$3 DO_SHUTDOWN=$4 TERMINATE=$TERMINATE STACKSH_PARAMS="$COMMON_VARS $5" ./build_domU.sh
+}
+
+# Launch the head node - headnode uses a non-ip domain name,
+# because rabbit won't launch with an ip addr hostname :(
+install_domU HEADNODE $HEAD_PUB_IP $HEAD_MGT_IP 1 "ENABLED_SERVICES=g-api,g-reg,key,n-api,n-sch,n-vnc,horizon,mysql,rabbit"
+
+if [ $HEAD_PUB_IP == "dhcp" ]
+then
+    guestnet=$(xe vm-list --minimal name-label=HEADNODE params=networks)
+    HEAD_PUB_IP=$(echo $guestnet | grep -w -o --only-matching "3/ip: [0-9,.]*;" | cut -d ':' -f2 | cut -d ';' -f 1)
+fi
+# Wait till the head node is up
+while ! curl -L http://$HEAD_PUB_IP | grep -q username; do
+    echo "Waiting for head node ($HEAD_PUB_IP) to start..."
+    sleep 5
+done
+
+# Build the HA compute host
+install_domU COMPUTENODE $COMPUTE_PUB_IP $COMPUTE_MGT_IP 0 "ENABLED_SERVICES=n-cpu,n-net,n-api"
--- a/tools/xen/prepare_dom0.sh
+++ b/tools/xen/prepare_dom0.sh
@ -34,8 +34,3 @@ if ! which git; then
    make install
 fi

-# Clone devstack
-DEVSTACK=/root/devstack
-if [ ! -d $DEVSTACK ]; then
-    git clone git://github.com/cloudbuilders/devstack.git $DEVSTACK
-fi
--- a/tools/xen/prepare_guest.sh
+++ b/tools/xen/prepare_guest.sh
@ -1,5 +1,7 @@
 #!/bin/bash

+set -x
+
 # Configurable nuggets
 GUEST_PASSWORD=${GUEST_PASSWORD:-secrete}
 STAGING_DIR=${STAGING_DIR:-stage}
--- a/tools/xen/scripts/install-os-vpx.sh
+++ b/tools/xen/scripts/install-os-vpx.sh
@ -38,7 +38,7 @@ usage()
 cat << EOF

  Usage: $0 [-f FILE_PATH] [-d DISK_SIZE] [-v BRIDGE_NAME] [-m BRIDGE_NAME] [-p BRIDGE_NAME]
-            [-k PARAMS] [-r RAM] [-i|-c] [-w] [-b]
+            [-k PARAMS] [-r RAM] [-i|-c] [-w] [-b] [-l NAME_LABEL]

  Installs XenServer OpenStack VPX.

@ -60,6 +60,7 @@ cat << EOF
     -k params    Specifies kernel parameters.
     -r MiB       Specifies RAM used by the VPX, in MiB.
                  By default it will take the value from the XVA.
+     -l name      Specifies the name label for the VM.

  EXAMPLES:

@ -87,7 +88,7 @@ EOF

 get_params()
 {
-  while getopts "hicwbf:d:v:m:p:k:r:" OPTION; 
+  while getopts "hicwbf:d:v:m:p:k:r:l:" OPTION; 
  do
    case $OPTION in
      h) usage
@ -126,6 +127,9 @@ get_params()
      v)
         BRIDGE_V=$OPTARG
         ;;
+     l)
+         NAME_LABEL=$OPTARG
+         ;;
      ?)
         usage
         exit
@ -443,7 +447,7 @@ else

  renumber_system_disk "$vm_uuid"

-  nl=$(xe_min vm-list params=name-label uuid=$vm_uuid)
+  nl=${NAME_LABEL:-$(xe_min vm-list params=name-label uuid=$vm_uuid)}
  xe vm-param-set \
    "name-label=${nl/ import/}" \
    other-config:os-vpx=true \
--- a/tools/xen/templates/interfaces.in
+++ b/tools/xen/templates/interfaces.in
@ -1,8 +1,15 @@
 auto lo
 iface lo inet loopback

-auto eth0
-iface eth0 inet dhcp
+# If eth3 is static, the order should not matter
+# and eth0 will have the default gateway. If not,
+# we probably want the default gateway to be
+# what is on the public interface. Hence changed
+# the order here.
+auto eth3
+iface eth3 inet static
+        address @ETH3_IP@
+        netmask @ETH3_NETMASK@

 auto eth1
 iface eth1 inet static
@ -15,7 +22,5 @@ iface eth2 inet static
        address @ETH2_IP@
        netmask @ETH2_NETMASK@

-auto eth3
-iface eth3 inet static
-        address @ETH3_IP@
-        netmask @ETH3_NETMASK@
+auto eth0
+iface eth0 inet dhcp
--- a/tools/xen/xenrc
+++ b/tools/xen/xenrc
@ -9,24 +9,31 @@ VDI_MB=${VDI_MB:-2500}
 # VM Password
 GUEST_PASSWORD=${GUEST_PASSWORD:-secrete}

-# Our nova host's network info 
+# Host Interface, i.e. the public facing interface on the nova vm
+HOST_IP_IFACE=${HOST_IP_IFACE:-eth0}
+
+# Our nova host's network info
 VM_IP=${VM_IP:-10.255.255.255} # A host-only ip that let's the interface come up, otherwise unused
 MGT_IP=${MGT_IP:-172.16.100.55}
 PUB_IP=${PUB_IP:-192.168.1.55}

 # Public network
-PUB_BR=${PUB_BR:-xenbr0}
+PUB_BR=${PUB_BR:-"xenbr0"}
+PUB_DEV=${PUB_DEV:-eth0}
+PUB_VLAN=${PUB_VLAN:--1}
 PUB_NETMASK=${PUB_NETMASK:-255.255.255.0}

 # VM network params
 VM_NETMASK=${VM_NETMASK:-255.255.255.0}
-VM_BR=${VM_BR:-xapi1}
+VM_BR=${VM_BR:-""}
 VM_VLAN=${VM_VLAN:-100}
+VM_DEV=${VM_DEV:-eth0}

 # MGMT network params
 MGT_NETMASK=${MGT_NETMASK:-255.255.255.0}
-MGT_BR=${MGT_BR:-xapi2}
+MGT_BR=${MGT_BR:-""}
 MGT_VLAN=${MGT_VLAN:-101}
+MGT_DEV=${MGT_DEV:-eth0}

 # XVA Directory
 XVA_DIR=${XVA_DIR:-xvas}