From 442e4e962559479fa0000ad64e6fa34da2c141c8 Mon Sep 17 00:00:00 2001
From: Sean Dague <sean@dague.net>
Date: Wed, 24 Jun 2015 13:24:02 -0400
Subject: [PATCH] make test_with_retry a function

We have this pattern of timeout with while tests for a non infinite
while loop condition. It's enough of a pattern that we should probably
extract it into a function to make it more widely used.

Change-Id: I11afcda9fac9709acf2f52d256d6e97644d4727c
---
 functions-common   | 13 +++++++++++++
 lib/neutron-legacy | 12 ++++++------
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/functions-common b/functions-common
index c85052d5b0..061a9356f5 100644
--- a/functions-common
+++ b/functions-common
@@ -1967,6 +1967,19 @@ function stop_service {
     fi
 }
 
+# Test with a finite retry loop.
+#
+function test_with_retry {
+    local testcmd=$1
+    local failmsg=$2
+    local until=${3:-10}
+    local sleep=${4:-0.5}
+
+    if ! timeout $until sh -c "while ! $testcmd; do sleep $sleep; done"; then
+        die $LINENO "$failmsg"
+    fi
+}
+
 
 # Restore xtrace
 $XTRACE
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 3ac76a2586..4cfb026ae2 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -696,9 +696,10 @@ function start_neutron_service_and_check {
     if is_ssl_enabled_service "neutron"; then
         ssl_ca="--ca-certificate=${SSL_BUNDLE_FILE}"
     fi
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port; do sleep 1; done"; then
-        die $LINENO "Neutron did not start"
-    fi
+
+    local testcmd="wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port"
+    test_with_retry "$testcmd" "Neutron did not start" $SERVICE_TIMEOUT
+
     # Start proxy if enabled
     if is_service_enabled tls-proxy; then
         start_tls_proxy '*' $Q_PORT $Q_HOST $Q_PORT_INT &
@@ -1380,9 +1381,8 @@ function _ssh_check_neutron {
     local timeout_sec=$5
     local probe_cmd = ""
     probe_cmd=`_get_probe_cmd_prefix $from_net`
-    if ! timeout $timeout_sec sh -c "while ! $probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success; do sleep 1; done"; then
-        die $LINENO "server didn't become ssh-able!"
-    fi
+    local testcmd="$probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success"
+    test_with_retry "$testcmd" "server $ip didn't become ssh-able" $timeout_sec
 }
 
 # Neutron 3rd party programs