Add stack phases to extras.d handling

Add hooks to stack.sh, unstack.sh and clean.sh to call the extras.d
scripts at multiple points in stack.sh.  This allows these scripts to
perform installation and startup tasks at similar times as they would
if integrated into stack.sh.

extras.d/70-tempest.sh is present as an example of the structure
of these scripts.

See extras.d/README.md for more information.

Change-Id: Ic1fe522559b94d204d6c0319a2e3d23684c8d028

README.md

@@ -215,6 +215,10 @@ If tempest has been successfully configured, a basic set of smoke tests can be r
     $ cd /opt/stack/tempest
     $ nosetests tempest/scenario/test_network_basic_ops.py
 
+# Additional Projects
+
+DevStack has a hook mechanism to call out to a dispatch script at specific points in the execution if `stack.sh`, `unstack.sh` and `clean.sh`.  This allows higher-level projects, especially those that the lower level projects have no dependency on, to be added to DevStack without modifying the scripts.  Tempest is built this way as an example of how to structure the dispatch script, see `extras.d/80-tempest.sh`.  See `extras.d/README.md` for more information.
+
 # Multi-Node Setup
 
 A more interesting setup involves running multiple compute nodes, with Neutron networks connecting VMs on different compute nodes.

clean.sh

@@ -47,6 +47,15 @@ source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/baremetal
 source $TOP_DIR/lib/ldap
 
+# Extras Source
+# --------------
+
+# Phase: source
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i source
+    done
+fi
 
 # See if there is anything running...
 # need to adapt when run_service is merged
@@ -56,6 +65,16 @@ if [[ -n "$SESSION" ]]; then
     $TOP_DIR/unstack.sh --all
 fi
 
+# Run extras
+# ==========
+
+# Phase: clean
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i clean
+    done
+fi
+
 # Clean projects
 cleanup_oslo
 cleanup_cinder

extras.d/80-tempest.sh

@@ -1,21 +1,29 @@
 # tempest.sh - DevStack extras script
 
-source $TOP_DIR/lib/tempest
-
-if [[ "$1" == "stack" ]]; then
-    # Configure Tempest last to ensure that the runtime configuration of
-    # the various OpenStack services can be queried.
-    if is_service_enabled tempest; then
-        echo_summary "Configuring Tempest"
+if is_service_enabled tempest; then
+    if [[ "$1" == "source" ]]; then
+        # Initial source
+        source $TOP_DIR/lib/tempest
+    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
+        echo_summary "Installing Tempest"
         install_tempest
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+        # Tempest config must come after layer 2 services are running
+        :
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        echo_summary "Initializing Tempest"
         configure_tempest
         init_tempest
     fi
+
+    if [[ "$1" == "unstack" ]]; then
+        # no-op
+        :
+    fi
+
+    if [[ "$1" == "clean" ]]; then
+        # no-op
+        :
+    fi
 fi
 
-if [[ "$1" == "unstack" ]]; then
-    # no-op
-    :
-fi
-
-

extras.d/README

@@ -1,14 +0,0 @@
-The extras.d directory contains project initialization scripts to be
-sourced by stack.sh at the end of its run.  This is expected to be
-used by external projects that want to be configured, started and
-stopped with DevStack.
-
-Order is controlled by prefixing the script names with the a two digit
-sequence number.  Script names must end with '.sh'.  This provides a
-convenient way to disable scripts by simoy renaming them.
-
-DevStack reserves the sequence numbers 00 through 09 and 90 through 99
-for its own use.
-
-The scripts are called with an argument of 'stack' by stack.sh and
-with an argument of 'unstack' by unstack.sh.

extras.d/README.md

@@ -0,0 +1,31 @@
+# Extras Hooks
+
+The `extras.d` directory contains project dispatch scripts that are called
+at specific times by `stack.sh`, `unstack.sh` and `clean.sh`.  These hooks are
+used to install, configure and start additional projects during a DevStack run
+without any modifications to the base DevStack scripts.
+
+When `stack.sh` reaches one of the hook points it sources the scripts in `extras.d`
+that end with `.sh`.  To control the order that the scripts are sourced their
+names start with a two digit sequence number.  DevStack reserves the sequence
+numbers 00 through 09 and 90 through 99 for its own use.
+
+The scripts are sourced at each hook point so they should not declare anything
+at the top level that would cause a problem, specifically, functions.  This does
+allow the entire `stack.sh` variable space to be available.  The scripts are
+sourced with one or more arguments, the first of which defines the hook phase:
+
+arg 1: source | stack | unstack | clean
+
+    source: always called first in any of the scripts, used to set the
+        initial defaults in a lib/* script or similar
+
+    stack: called by stack.sh.  There are three possible values for
+        the second arg to distinguish the phase stack.sh is in:
+
+        arg 2:  install | post-config | extra
+
+    unstack: called by unstack.sh
+
+    clean: called by clean.sh.  Remember, clean.sh also calls unstack.sh
+        so that work need not be repeated.

stack.sh

@@ -313,6 +313,16 @@ source $TOP_DIR/lib/ldap
 source $TOP_DIR/lib/ironic
 source $TOP_DIR/lib/trove
 
+# Extras Source
+# --------------
+
+# Phase: source
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i source
+    done
+fi
+
 # Set the destination directories for other OpenStack projects
 OPENSTACKCLIENT_DIR=$DEST/python-openstackclient
 
@@ -725,6 +735,16 @@ if is_service_enabled ir-api ir-cond; then
     configure_ironic
 fi
 
+# Extras Install
+# --------------
+
+# Phase: install
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i stack install
+    done
+fi
+
 if [[ $TRACK_DEPENDS = True ]]; then
     $DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
     if ! diff -Nru $DEST/requires-pre-pip $DEST/requires-post-pip > $DEST/requires.diff; then
@@ -1000,6 +1020,17 @@ if is_service_enabled nova && is_baremetal; then
 fi
 
 
+# Extras Configuration
+# ====================
+
+# Phase: post-config
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i stack post-config
+    done
+fi
+
+
 # Local Configuration
 # ===================
 
@@ -1214,9 +1245,10 @@ merge_config_group $TOP_DIR/local.conf extra
 # Run extras
 # ==========
 
+# Phase: extra
 if [[ -d $TOP_DIR/extras.d ]]; then
     for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i stack
+        [[ -r $i ]] && source $i stack extra
     done
 fi
 

unstack.sh

@@ -42,6 +42,16 @@ source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/ironic
 source $TOP_DIR/lib/trove
 
+# Extras Source
+# --------------
+
+# Phase: source
+if [[ -d $TOP_DIR/extras.d ]]; then
+    for i in $TOP_DIR/extras.d/*.sh; do
+        [[ -r $i ]] && source $i source
+    done
+fi
+
 # Determine what system we are running on.  This provides ``os_VENDOR``,
 # ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
 GetOSVersion
@@ -53,6 +63,7 @@ fi
 # Run extras
 # ==========
 
+# Phase: unstack
 if [[ -d $TOP_DIR/extras.d ]]; then
     for i in $TOP_DIR/extras.d/*.sh; do
         [[ -r $i ]] && source $i unstack