Revert "Build retry loop for screen sessions"
This reverts commit 0afa912e99.
This possibly made things worse, though it times in with the
trusty add, so it's hard to tell. Revert to see if grenade gets
better.
Change-Id: Ic399957fc9d4a7da28b030cdf895df061b2567c8
Related-Bug: #1331274
This commit is contained in:
Sean Dague
parent
bfa5817c50
commit
ea22a4fdba
112
functions-common
112
functions-common
@ -1058,100 +1058,44 @@ function run_process {
|
||||
echo $!
|
||||
}
|
||||
|
||||
function _start_in_screen {
|
||||
local service=$1
|
||||
local cmd=$2
|
||||
local screen_name=${SCREEN_NAME:-stack}
|
||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
||||
local service_dir="$status_dir/$screen_name"
|
||||
local pid="$service_dir/$service.pid"
|
||||
local failure="$service_dir/$service.failure"
|
||||
|
||||
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
||||
local logfile=${SCREEN_LOGDIR}/screen-${service}.${CURRENT_LOG_TIME}.log
|
||||
local shortlog=${SCREEN_LOGDIR}/screen-${service}.log
|
||||
# this whole dance is done because of slow nodes
|
||||
screen -S $screen_name -p $service -X logfile ${logfile}
|
||||
screen -S $screen_name -p $service -X log on
|
||||
ln -sf ${logfile} ${shortlog}
|
||||
fi
|
||||
|
||||
NL=`echo -ne '\015'`
|
||||
# This fun command does the following:
|
||||
# - the passed server command is backgrounded
|
||||
# - the pid of the background process is saved in the usual place
|
||||
# - the server process is brought back to the foreground
|
||||
# - if the server process exits prematurely the fg command errors
|
||||
# and a message is written to stdout and the service failure file
|
||||
# The pid saved can be used in screen_stop() as a process group
|
||||
# id to kill off all child processes
|
||||
echo "Running: $cmd & echo \$! >$pid; fg || echo \"$service failed to start\" | tee \"$failure\"$NL"
|
||||
screen -S $screen_name -p $service -X stuff "$cmd & echo \$! >$pid; fg || echo \"$service failed to start\" | tee \"$failure\"$NL"
|
||||
}
|
||||
|
||||
|
||||
function _is_running_in_screen {
|
||||
local service=$1
|
||||
local screen_name=${SCREEN_NAME:-stack}
|
||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
||||
local service_dir="$status_dir/$screen_name"
|
||||
local pid="$service_dir/$service.pid"
|
||||
local failure="$service_dir/$service.failure"
|
||||
if [[ ! -e "$pid" && ! -e "$failure" ]]; then
|
||||
# if we don't have a pid or a failure for why, the command may not
|
||||
# have stuffed in there
|
||||
echo "Warning: neither $pid nor $failure exist, $service didn't seem to start"
|
||||
return 1
|
||||
fi
|
||||
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
||||
# if we should be logging, but we don't have a log file, something is wrong
|
||||
local logfile=${SCREEN_LOGDIR}/screen-${service}.${CURRENT_LOG_TIME}.log
|
||||
if [[ ! -e "$logfile" ]]; then
|
||||
echo "Warning: expected logfile $logfile not found, something wrong with starting $service"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Helper to launch a service in a named screen
|
||||
# screen_it service "command-line"
|
||||
function screen_it {
|
||||
local service=$1
|
||||
local cmd=$2
|
||||
local screen_name=${SCREEN_NAME:-stack}
|
||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
||||
local service_dir="$status_dir/$screen_name"
|
||||
local use_screen=$(trueorfalse True $USE_SCREEN)
|
||||
local pid="$service_dir/$service.pid"
|
||||
SCREEN_NAME=${SCREEN_NAME:-stack}
|
||||
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
|
||||
USE_SCREEN=$(trueorfalse True $USE_SCREEN)
|
||||
|
||||
if is_service_enabled $1; then
|
||||
# Append the service to the screen rc file
|
||||
screen_rc "$service" "$cmd"
|
||||
screen_rc "$1" "$2"
|
||||
|
||||
if [[ "$use_screen" = "True" ]]; then
|
||||
screen -S $screen_name -X screen -t $service
|
||||
if [[ "$USE_SCREEN" = "True" ]]; then
|
||||
screen -S $SCREEN_NAME -X screen -t $1
|
||||
|
||||
# this retry loop brought to you by slow compute nodes, screen raciness,
|
||||
# and frustration in upgrading.
|
||||
local screen_tries=0
|
||||
while [ "$screen_tries" -lt 10 ]; do
|
||||
_start_in_screen "$service" "$cmd"
|
||||
if _is_running_in_screen $service; then
|
||||
screen_tries=10
|
||||
else
|
||||
screen_tries=$[screen_tries + 1]
|
||||
echo "Failed to start service after $screen_tries attempt(s), retrying"
|
||||
if [[ "$screen_tries" -eq 10 ]]; then
|
||||
echo "Too many retries, giving up"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
||||
screen -S $SCREEN_NAME -p $1 -X logfile ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log
|
||||
screen -S $SCREEN_NAME -p $1 -X log on
|
||||
ln -sf ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log ${SCREEN_LOGDIR}/screen-${1}.log
|
||||
fi
|
||||
|
||||
# sleep to allow bash to be ready to be send the command - we are
|
||||
# creating a new window in screen and then sends characters, so if
|
||||
# bash isn't running by the time we send the command, nothing happens
|
||||
sleep 3
|
||||
|
||||
NL=`echo -ne '\015'`
|
||||
# This fun command does the following:
|
||||
# - the passed server command is backgrounded
|
||||
# - the pid of the background process is saved in the usual place
|
||||
# - the server process is brought back to the foreground
|
||||
# - if the server process exits prematurely the fg command errors
|
||||
# and a message is written to stdout and the service failure file
|
||||
# The pid saved can be used in screen_stop() as a process group
|
||||
# id to kill off all child processes
|
||||
screen -S $SCREEN_NAME -p $1 -X stuff "$2 & echo \$! >$SERVICE_DIR/$SCREEN_NAME/$1.pid; fg || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
|
||||
else
|
||||
# Spawn directly without screen
|
||||
run_process "$service" "$cmd" >$pid
|
||||
run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user