From 489bd2a62b5949665bc7c4a05a52d27a987e0489 Mon Sep 17 00:00:00 2001
From: Dean Troyer <dtroyer@gmail.com>
Date: Fri, 2 Mar 2012 10:44:29 -0600
Subject: [PATCH] Improve exercise robustness

* Test returns and exit codes on most command invocations
* Add start and end banners to make output easier to find in
  long log files
* Adds die_if_error(), die_if_not_set() and is_set() to functions
* Add some function tests

Fixes bug 944593

Change-Id: I55e2962c5fec9aad237b674732b1e922ad37a62e
---
 exercises/bundle.sh       | 24 ++++++++++++++---
 exercises/client-env.sh   | 35 ++++++++++---------------
 exercises/euca.sh         | 28 +++++++++++++++++---
 exercises/floating_ips.sh | 42 +++++++++++++++++++++++-------
 exercises/swift.sh        | 34 +++++++++++++++++++-----
 exercises/volumes.sh      | 48 +++++++++++++++++++++-------------
 functions                 | 54 +++++++++++++++++++++++++++++++++++++++
 tests/functions.sh        | 54 +++++++++++++++++++++++++++++++++++++++
 8 files changed, 259 insertions(+), 60 deletions(-)
 create mode 100755 tests/functions.sh

diff --git a/exercises/bundle.sh b/exercises/bundle.sh
index d5c78af32b..e1c949cf47 100755
--- a/exercises/bundle.sh
+++ b/exercises/bundle.sh
@@ -2,7 +2,10 @@
 
 # we will use the ``euca2ools`` cli tool that wraps the python boto
 # library to test ec2 compatibility
-#
+
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
 
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
@@ -16,7 +19,12 @@ set -o xtrace
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
 
 # Remove old certificates
@@ -27,7 +35,7 @@ rm -f pk.pem
 # Get Certificates
 nova x509-get-root-cert
 nova x509-create-cert
-popd
+popd >/dev/null
 
 # Max time to wait for image to be registered
 REGISTER_TIMEOUT=${REGISTER_TIMEOUT:-15}
@@ -36,10 +44,14 @@ BUCKET=testbucket
 IMAGE=bundle.img
 truncate -s 5M /tmp/$IMAGE
 euca-bundle-image -i /tmp/$IMAGE
+die_if_error "Failure bundling image $IMAGE"
 
 
 euca-upload-bundle -b $BUCKET -m /tmp/$IMAGE.manifest.xml
+die_if_error "Failure uploading bundle $IMAGE to $BUCKET"
+
 AMI=`euca-register $BUCKET/$IMAGE.manifest.xml | cut -f2`
+die_if_not_set AMI "Failure registering $BUCKET/$IMAGE"
 
 # Wait for the image to become available
 if ! timeout $REGISTER_TIMEOUT sh -c "while euca-describe-images | grep '$AMI' | grep 'available'; do sleep 1; done"; then
@@ -49,3 +61,9 @@ fi
 
 # Clean up
 euca-deregister $AMI
+die_if_error "Failure deregistering $AMI"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
diff --git a/exercises/client-env.sh b/exercises/client-env.sh
index a15a5c04d9..28c4d95e00 100755
--- a/exercises/client-env.sh
+++ b/exercises/client-env.sh
@@ -2,6 +2,10 @@
 
 # Test OpenStack client enviroment variable handling
 
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
+
 # Verify client workage
 VERIFY=${1:-""}
 
@@ -10,6 +14,11 @@ VERIFY=${1:-""}
 
 # Use openrc + stackrc + localrc for settings
 pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
 popd >/dev/null
 
@@ -23,19 +32,10 @@ unset NOVA_URL
 unset NOVA_USERNAME
 unset NOVA_VERSION
 
-# Make sure we have the vars we are expecting
-function is_set() {
-    local var=\$"$1"
-    eval echo $1=$var
-    if eval "[ -z $var ]"; then
-        return 1
-    fi
-    return 0
-}
-
 for i in OS_TENANT_NAME OS_USERNAME OS_PASSWORD OS_AUTH_URL; do
     is_set $i
     if [[ $? -ne 0 ]]; then
+        echo "$i expected to be set"
         ABORT=1
     fi
 done
@@ -52,14 +52,6 @@ if [[ "$ENABLED_SERVICES" =~ "key" ]]; then
     if [[ "$SKIP_EXERCISES" =~ "key" ]] ; then
         STATUS_KEYSTONE="Skipped"
     else
-        # We need to run the keystone test as admin since there doesn't
-        # seem to be anything to test the cli vars that runs as a user
-        # tenant-list should do that, it isn't implemented (yet)
-        xOS_TENANT_NAME=$OS_TENANT_NAME
-        xOS_USERNAME=$OS_USERNAME
-        export OS_USERNAME=admin
-        export OS_TENANT_NAME=admin
-
         echo -e "\nTest Keystone"
         if keystone service-list; then
             STATUS_KEYSTONE="Succeeded"
@@ -67,9 +59,6 @@ if [[ "$ENABLED_SERVICES" =~ "key" ]]; then
             STATUS_KEYSTONE="Failed"
             RETURN=1
         fi
-
-        OS_TENANT_NAME=$xOS_TENANT_NAME
-        OS_USERNAME=$xOS_USERNAME
     fi
 fi
 
@@ -139,4 +128,8 @@ report "Nova" $STATUS_NOVA
 report "Glance" $STATUS_GLANCE
 report "Swift" $STATUS_SWIFT
 
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
+
 exit $RETURN
diff --git a/exercises/euca.sh b/exercises/euca.sh
index 86cd67321e..b766bab8b5 100755
--- a/exercises/euca.sh
+++ b/exercises/euca.sh
@@ -2,7 +2,10 @@
 
 # we will use the ``euca2ools`` cli tool that wraps the python boto
 # library to test ec2 compatibility
-#
+
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
 
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
@@ -16,9 +19,14 @@ set -o xtrace
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
-popd
+popd >/dev/null
 
 # Max time to wait while vm goes from build to active state
 ACTIVE_TIMEOUT=${ACTIVE_TIMEOUT:-30}
@@ -49,6 +57,7 @@ fi
 
 # Launch it
 INSTANCE=`euca-run-instances -g $SECGROUP -t $DEFAULT_INSTANCE_TYPE $IMAGE | grep INSTANCE | cut -f2`
+die_if_not_set INSTANCE "Failure launching instance"
 
 # Assure it has booted within a reasonable time
 if ! timeout $RUNNING_TIMEOUT sh -c "while ! euca-describe-instances $INSTANCE | grep -q running; do sleep 1; done"; then
@@ -58,12 +67,15 @@ fi
 
 # Allocate floating address
 FLOATING_IP=`euca-allocate-address | cut -f2`
+die_if_not_set FLOATING_IP "Failure allocating floating IP"
 
 # Associate floating address
 euca-associate-address -i $INSTANCE $FLOATING_IP
+die_if_error "Failure associating address $FLOATING_IP to $INSTANCE"
 
 # Authorize pinging
 euca-authorize -P icmp -s 0.0.0.0/0 -t -1:-1 $SECGROUP
+die_if_error "Failure authorizing rule in $SECGROUP"
 
 # Test we can ping our floating ip within ASSOCIATE_TIMEOUT seconds
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! ping -c1 -w1 $FLOATING_IP; do sleep 1; done"; then
@@ -73,9 +85,11 @@ fi
 
 # Revoke pinging
 euca-revoke -P icmp -s 0.0.0.0/0 -t -1:-1 $SECGROUP
+die_if_error "Failure revoking rule in $SECGROUP"
 
 # Release floating address
 euca-disassociate-address $FLOATING_IP
+die_if_error "Failure disassociating address $FLOATING_IP"
 
 # Wait just a tick for everything above to complete so release doesn't fail
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while euca-describe-addresses | grep $INSTANCE | grep -q $FLOATING_IP; do sleep 1; done"; then
@@ -85,6 +99,7 @@ fi
 
 # Release floating address
 euca-release-address $FLOATING_IP
+die_if_error "Failure releasing address $FLOATING_IP"
 
 # Wait just a tick for everything above to complete so terminate doesn't fail
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while euca-describe-addresses | grep -q $FLOATING_IP; do sleep 1; done"; then
@@ -94,6 +109,7 @@ fi
 
 # Terminate instance
 euca-terminate-instances $INSTANCE
+die_if_error "Failure terminating instance $INSTANCE"
 
 # Assure it has terminated within a reasonable time
 if ! timeout $TERMINATE_TIMEOUT sh -c "while euca-describe-instances $INSTANCE | grep -q running; do sleep 1; done"; then
@@ -103,3 +119,9 @@ fi
 
 # Delete group
 euca-delete-group $SECGROUP
+die_if_error "Failure deleting security group $SECGROUP"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
diff --git a/exercises/floating_ips.sh b/exercises/floating_ips.sh
index b559965fc9..a47f1ffc23 100755
--- a/exercises/floating_ips.sh
+++ b/exercises/floating_ips.sh
@@ -7,6 +7,10 @@
 #
 
 
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
+
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
 set -o errexit
@@ -20,9 +24,14 @@ set -o xtrace
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
-popd
+popd >/dev/null
 
 # Max time to wait while vm goes from build to active state
 ACTIVE_TIMEOUT=${ACTIVE_TIMEOUT:-30}
@@ -87,15 +96,16 @@ fi
 # List of instance types:
 nova flavor-list
 
-INSTANCE_TYPE=`nova flavor-list | grep $DEFAULT_INSTANCE_TYPE | cut -d"|" -f2`
+INSTANCE_TYPE=`nova flavor-list | grep $DEFAULT_INSTANCE_TYPE | get_field 1`
 if [[ -z "$INSTANCE_TYPE" ]]; then
     # grab the first flavor in the list to launch if default doesn't exist
-   INSTANCE_TYPE=`nova flavor-list | head -n 4 | tail -n 1 | cut -d"|" -f2`
+   INSTANCE_TYPE=`nova flavor-list | head -n 4 | tail -n 1 | get_field 1`
 fi
 
-NAME="myserver"
+NAME="ex-float"
 
-VM_UUID=`nova boot --flavor $INSTANCE_TYPE --image $IMAGE $NAME --security_groups=$SECGROUP | grep ' id ' | cut -d"|" -f3 | sed 's/ //g'`
+VM_UUID=`nova boot --flavor $INSTANCE_TYPE --image $IMAGE $NAME --security_groups=$SECGROUP | grep ' id ' | get_field 2`
+die_if_not_set VM_UUID "Failure launching $NAME"
 
 # Testing
 # =======
@@ -114,7 +124,8 @@ if ! timeout $ACTIVE_TIMEOUT sh -c "while ! nova show $VM_UUID | grep status | g
 fi
 
 # get the IP of the server
-IP=`nova show $VM_UUID | grep "private network" | cut -d"|" -f3`
+IP=`nova show $VM_UUID | grep "private network" | get_field 2`
+die_if_not_set IP "Failure retrieving IP address"
 
 # for single node deployments, we can ping private ips
 MULTI_HOST=${MULTI_HOST:-0}
@@ -147,7 +158,8 @@ fi
 nova secgroup-list-rules $SECGROUP
 
 # allocate a floating ip from default pool
-FLOATING_IP=`nova floating-ip-create | grep $DEFAULT_FLOATING_POOL | cut -d '|' -f2`
+FLOATING_IP=`nova floating-ip-create | grep $DEFAULT_FLOATING_POOL | get_field 1`
+die_if_not_set FLOATING_IP "Failure creating floating IP"
 
 # list floating addresses
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! nova floating-ip-list | grep -q $FLOATING_IP; do sleep 1; done"; then
@@ -157,6 +169,7 @@ fi
 
 # add floating ip to our server
 nova add-floating-ip $VM_UUID $FLOATING_IP
+die_if_error "Failure adding floating IP $FLOATING_IP to $NAME"
 
 # test we can ping our floating ip within ASSOCIATE_TIMEOUT seconds
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! ping -c1 -w1 $FLOATING_IP; do sleep 1; done"; then
@@ -165,7 +178,8 @@ if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! ping -c1 -w1 $FLOATING_IP; do sle
 fi
 
 # Allocate an IP from second floating pool
-TEST_FLOATING_IP=`nova floating-ip-create $TEST_FLOATING_POOL | grep $TEST_FLOATING_POOL | cut -d '|' -f2`
+TEST_FLOATING_IP=`nova floating-ip-create $TEST_FLOATING_POOL | grep $TEST_FLOATING_POOL | get_field 1`
+die_if_not_set TEST_FLOATING_IP "Failure creating floating IP in $TEST_FLOATING_POOL"
 
 # list floating addresses
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! nova floating-ip-list | grep $TEST_FLOATING_POOL | grep -q $TEST_FLOATING_IP; do sleep 1; done"; then
@@ -175,6 +189,7 @@ fi
 
 # dis-allow icmp traffic (ping)
 nova secgroup-delete-rule $SECGROUP icmp -1 -1 0.0.0.0/0
+die_if_error "Failure deleting security group rule from $SECGROUP"
 
 # FIXME (anthony): make xs support security groups
 if [ "$VIRT_DRIVER" != "xenserver" ]; then
@@ -188,12 +203,15 @@ fi
 
 # de-allocate the floating ip
 nova floating-ip-delete $FLOATING_IP
+die_if_error "Failure deleting floating IP $FLOATING_IP"
 
 # Delete second floating IP
 nova floating-ip-delete $TEST_FLOATING_IP
+die_if_error "Failure deleting floating IP $TEST_FLOATING_IP"
 
 # shutdown the server
 nova delete $VM_UUID
+die_if_error "Failure deleting instance $NAME"
 
 # make sure the VM shuts down within a reasonable time
 if ! timeout $TERMINATE_TIMEOUT sh -c "while nova show $VM_UUID | grep status | grep -q ACTIVE; do sleep 1; done"; then
@@ -203,3 +221,9 @@ fi
 
 # Delete a secgroup
 nova secgroup-delete $SECGROUP
+die_if_error "Failure deleting security group $SECGROUP"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
diff --git a/exercises/swift.sh b/exercises/swift.sh
index 95443df33d..76096379a4 100755
--- a/exercises/swift.sh
+++ b/exercises/swift.sh
@@ -2,6 +2,10 @@
 
 # Test swift via the command line tools that ship with it.
 
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
+
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
 set -o errexit
@@ -15,9 +19,17 @@ set -o xtrace
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
-popd
+popd >/dev/null
+
+# Container name
+CONTAINER=ex-swift
 
 
 # Testing Swift
@@ -25,16 +37,26 @@ popd
 
 # Check if we have to swift via keystone
 swift stat
+die_if_error "Failure geting status"
 
 # We start by creating a test container
-swift post testcontainer
+swift post $CONTAINER
+die_if_error "Failure creating container $CONTAINER"
 
 # add some files into it.
-swift upload testcontainer /etc/issue
+swift upload $CONTAINER /etc/issue
+die_if_error "Failure uploading file to container $CONTAINER"
 
 # list them
-swift list testcontainer
+swift list $CONTAINER
+die_if_error "Failure listing contents of container $CONTAINER"
 
 # And we may want to delete them now that we have tested that
 # everything works.
-swift delete testcontainer
+swift delete $CONTAINER
+die_if_error "Failure deleting container $CONTAINER"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
diff --git a/exercises/volumes.sh b/exercises/volumes.sh
index 622fb18586..a812401a09 100755
--- a/exercises/volumes.sh
+++ b/exercises/volumes.sh
@@ -2,6 +2,10 @@
 
 # Test nova volumes with the nova command from python-novaclient
 
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
+
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
 set -o errexit
@@ -15,9 +19,14 @@ set -o xtrace
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
-popd
+popd >/dev/null
 
 # Max time to wait while vm goes from build to active state
 ACTIVE_TIMEOUT=${ACTIVE_TIMEOUT:-30}
@@ -55,21 +64,6 @@ IMAGE=`glance -f index | egrep $DEFAULT_IMAGE_NAME | head -1 | cut -d" " -f1`
 # determinine instance type
 # -------------------------
 
-# Helper function to grab a numbered field from python novaclient cli result
-# Fields are numbered starting with 1
-# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
-function get_field () {
-    while read data
-    do
-        if [ "$1" -lt 0 ]; then
-            field="(\$(NF$1))"
-        else
-            field="\$$(($1 + 1))"
-        fi
-        echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
-    done
-}
-
 # List of instance types:
 nova flavor-list
 
@@ -79,9 +73,11 @@ if [[ -z "$INSTANCE_TYPE" ]]; then
    INSTANCE_TYPE=`nova flavor-list | head -n 4 | tail -n 1 | get_field 1`
 fi
 
-NAME="myserver"
+NAME="ex-vol"
 
 VM_UUID=`nova boot --flavor $INSTANCE_TYPE --image $IMAGE $NAME --security_groups=$SECGROUP | grep ' id ' | get_field 2`
+die_if_not_set VM_UUID "Failure launching $NAME"
+
 
 # Testing
 # =======
@@ -101,6 +97,7 @@ fi
 
 # get the IP of the server
 IP=`nova show $VM_UUID | grep "private network" | get_field 2`
+die_if_not_set IP "Failure retrieving IP address"
 
 # for single node deployments, we can ping private ips
 MULTI_HOST=${MULTI_HOST:-0}
@@ -130,6 +127,10 @@ fi
 
 # Create a new volume
 nova volume-create --display_name $VOL_NAME --display_description "test volume: $VOL_NAME" 1
+if [[ $? != 0 ]]; then
+    echo "Failure creating volume $VOL_NAME"
+    exit 1
+fi
 if ! timeout $ACTIVE_TIMEOUT sh -c "while ! nova volume-list | grep $VOL_NAME | grep available; do sleep 1; done"; then
     echo "Volume $VOL_NAME not created"
     exit 1
@@ -137,16 +138,19 @@ fi
 
 # Get volume ID
 VOL_ID=`nova volume-list | grep $VOL_NAME | head -1 | get_field 1`
+die_if_not_set VOL_ID "Failure retrieving volume ID for $VOL_NAME"
 
 # Attach to server
 DEVICE=/dev/vdb
 nova volume-attach $VM_UUID $VOL_ID $DEVICE
+die_if_error "Failure attaching volume $VOL_NAME to $NAME"
 if ! timeout $ACTIVE_TIMEOUT sh -c "while ! nova volume-list | grep $VOL_NAME | grep in-use; do sleep 1; done"; then
     echo "Volume $VOL_NAME not attached to $NAME"
     exit 1
 fi
 
 VOL_ATTACH=`nova volume-list | grep $VOL_NAME | head -1 | get_field -1`
+die_if_not_set VOL_ATTACH "Failure retrieving $VOL_NAME status"
 if [[ "$VOL_ATTACH" != $VM_UUID ]]; then
     echo "Volume not attached to correct instance"
     exit 1
@@ -154,6 +158,7 @@ fi
 
 # Detach volume
 nova volume-detach $VM_UUID $VOL_ID
+die_if_error "Failure detaching volume $VOL_NAME from $NAME"
 if ! timeout $ACTIVE_TIMEOUT sh -c "while ! nova volume-list | grep $VOL_NAME | grep available; do sleep 1; done"; then
     echo "Volume $VOL_NAME not detached from $NAME"
     exit 1
@@ -161,6 +166,7 @@ fi
 
 # Delete volume
 nova volume-delete $VOL_ID
+die_if_error "Failure deleting volume $VOL_NAME"
 if ! timeout $ACTIVE_TIMEOUT sh -c "while ! nova volume-list | grep $VOL_NAME; do sleep 1; done"; then
     echo "Volume $VOL_NAME not deleted"
     exit 1
@@ -168,3 +174,9 @@ fi
 
 # shutdown the server
 nova delete $NAME
+die_if_error "Failure deleting instance $NAME"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"
diff --git a/functions b/functions
index 01c4758fa0..adcf5bdaa1 100644
--- a/functions
+++ b/functions
@@ -22,6 +22,48 @@ function cp_it {
 }
 
 
+# Checks the exit code of the last command and prints "message"
+# if it is non-zero and exits
+# die_if_error "message"
+function die_if_error() {
+    local exitcode=$?
+    if [ $exitcode != 0 ]; then
+        echo $@
+        exit $exitcode
+    fi
+}
+
+
+# Checks an environment variable is not set or has length 0 OR if the
+# exit code is non-zero and prints "message" and exits
+# NOTE: env-var is the variable name without a '$'
+# die_if_not_set env-var "message"
+function die_if_not_set() {
+    local exitcode=$?
+    local evar=$1; shift
+    if ! is_set $evar || [ $exitcode != 0 ]; then
+        echo $@
+        exit 99
+    fi
+}
+
+
+# Grab a numbered field from python prettytable output
+# Fields are numbered starting with 1
+# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
+# get_field field-number
+function get_field() {
+    while read data; do
+        if [ "$1" -lt 0 ]; then
+            field="(\$(NF$1))"
+        else
+            field="\$$(($1 + 1))"
+        fi
+        echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
+    done
+}
+
+
 # git clone only if directory doesn't exist already.  Since ``DEST`` might not
 # be owned by the installation user, we create the directory and change the
 # ownership to the proper user.
@@ -67,6 +109,18 @@ function git_clone {
 }
 
 
+
+# Test if the named environment variable is set and not zero length
+# is_set env-var
+function is_set() {
+    local var=\$"$1"
+    if eval "[ -z $var ]"; then
+        return 1
+    fi
+    return 0
+}
+
+
 # pip install wrapper to set cache and proxy environment variables
 # pip_install package [package ...]
 function pip_install {
diff --git a/tests/functions.sh b/tests/functions.sh
new file mode 100755
index 0000000000..0fd76ccad0
--- /dev/null
+++ b/tests/functions.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Tests for DevStack functions
+
+TOP=$(cd $(dirname "$0")/.. && pwd)
+
+# Import common functions
+source $TOP/functions
+
+# Import configuration
+source $TOP/openrc
+
+
+echo "Testing die_if_error()"
+
+bash -c "source $TOP/functions; true; die_if_error 'not OK'"
+if [[ $? != 0 ]]; then
+    echo "die_if_error [true] Failed"
+fi
+
+bash -c "source $TOP/functions; false; die_if_error 'OK'"
+if [[ $? = 0 ]]; then
+    echo "die_if_error [false] Failed"
+else
+    echo 'OK'
+fi
+
+
+echo "Testing die_if_not_set()"
+
+bash -c "source $TOP/functions; X=`echo Y && true`; die_if_not_set X 'not OK'"
+if [[ $? != 0 ]]; then
+    echo "die_if_not_set [X='Y' true] Failed"
+else
+    echo 'OK'
+fi
+
+bash -c "source $TOP/functions; X=`true`; die_if_not_set X 'OK'"
+if [[ $? = 0 ]]; then
+    echo "die_if_not_set [X='' true] Failed"
+fi
+
+bash -c "source $TOP/functions; X=`echo Y && false`; die_if_not_set X 'not OK'"
+if [[ $? != 0 ]]; then
+    echo "die_if_not_set [X='Y' false] Failed"
+else
+    echo 'OK'
+fi
+
+bash -c "source $TOP/functions; X=`false`; die_if_not_set X 'OK'"
+if [[ $? = 0 ]]; then
+    echo "die_if_not_set [X='' false] Failed"
+fi
+