Merge "Revert "Build retry loop for screen sessions""
This commit is contained in:
commit
e33379658f
102
functions-common
102
functions-common
@ -1058,24 +1058,31 @@ function run_process {
|
|||||||
echo $!
|
echo $!
|
||||||
}
|
}
|
||||||
|
|
||||||
function _start_in_screen {
|
# Helper to launch a service in a named screen
|
||||||
local service=$1
|
# screen_it service "command-line"
|
||||||
local cmd=$2
|
function screen_it {
|
||||||
local screen_name=${SCREEN_NAME:-stack}
|
SCREEN_NAME=${SCREEN_NAME:-stack}
|
||||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
|
||||||
local service_dir="$status_dir/$screen_name"
|
USE_SCREEN=$(trueorfalse True $USE_SCREEN)
|
||||||
local pid="$service_dir/$service.pid"
|
|
||||||
local failure="$service_dir/$service.failure"
|
if is_service_enabled $1; then
|
||||||
|
# Append the service to the screen rc file
|
||||||
|
screen_rc "$1" "$2"
|
||||||
|
|
||||||
|
if [[ "$USE_SCREEN" = "True" ]]; then
|
||||||
|
screen -S $SCREEN_NAME -X screen -t $1
|
||||||
|
|
||||||
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
||||||
local logfile=${SCREEN_LOGDIR}/screen-${service}.${CURRENT_LOG_TIME}.log
|
screen -S $SCREEN_NAME -p $1 -X logfile ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log
|
||||||
local shortlog=${SCREEN_LOGDIR}/screen-${service}.log
|
screen -S $SCREEN_NAME -p $1 -X log on
|
||||||
# this whole dance is done because of slow nodes
|
ln -sf ${SCREEN_LOGDIR}/screen-${1}.${CURRENT_LOG_TIME}.log ${SCREEN_LOGDIR}/screen-${1}.log
|
||||||
screen -S $screen_name -p $service -X logfile ${logfile}
|
|
||||||
screen -S $screen_name -p $service -X log on
|
|
||||||
ln -sf ${logfile} ${shortlog}
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# sleep to allow bash to be ready to be send the command - we are
|
||||||
|
# creating a new window in screen and then sends characters, so if
|
||||||
|
# bash isn't running by the time we send the command, nothing happens
|
||||||
|
sleep 3
|
||||||
|
|
||||||
NL=`echo -ne '\015'`
|
NL=`echo -ne '\015'`
|
||||||
# This fun command does the following:
|
# This fun command does the following:
|
||||||
# - the passed server command is backgrounded
|
# - the passed server command is backgrounded
|
||||||
@ -1085,73 +1092,10 @@ function _start_in_screen {
|
|||||||
# and a message is written to stdout and the service failure file
|
# and a message is written to stdout and the service failure file
|
||||||
# The pid saved can be used in screen_stop() as a process group
|
# The pid saved can be used in screen_stop() as a process group
|
||||||
# id to kill off all child processes
|
# id to kill off all child processes
|
||||||
echo "Running: $cmd & echo \$! >$pid; fg || echo \"$service failed to start\" | tee \"$failure\"$NL"
|
screen -S $SCREEN_NAME -p $1 -X stuff "$2 & echo \$! >$SERVICE_DIR/$SCREEN_NAME/$1.pid; fg || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
|
||||||
screen -S $screen_name -p $service -X stuff "$cmd & echo \$! >$pid; fg || echo \"$service failed to start\" | tee \"$failure\"$NL"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function _is_running_in_screen {
|
|
||||||
local service=$1
|
|
||||||
local screen_name=${SCREEN_NAME:-stack}
|
|
||||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
|
||||||
local service_dir="$status_dir/$screen_name"
|
|
||||||
local pid="$service_dir/$service.pid"
|
|
||||||
local failure="$service_dir/$service.failure"
|
|
||||||
if [[ ! -e "$pid" && ! -e "$failure" ]]; then
|
|
||||||
# if we don't have a pid or a failure for why, the command may not
|
|
||||||
# have stuffed in there
|
|
||||||
echo "Warning: neither $pid nor $failure exist, $service didn't seem to start"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
if [[ -n ${SCREEN_LOGDIR} ]]; then
|
|
||||||
# if we should be logging, but we don't have a log file, something is wrong
|
|
||||||
local logfile=${SCREEN_LOGDIR}/screen-${service}.${CURRENT_LOG_TIME}.log
|
|
||||||
if [[ ! -e "$logfile" ]]; then
|
|
||||||
echo "Warning: expected logfile $logfile not found, something wrong with starting $service"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Helper to launch a service in a named screen
|
|
||||||
# screen_it service "command-line"
|
|
||||||
function screen_it {
|
|
||||||
local service=$1
|
|
||||||
local cmd=$2
|
|
||||||
local screen_name=${SCREEN_NAME:-stack}
|
|
||||||
local status_dir=${SERVICE_DIR:-${DEST}/status}
|
|
||||||
local service_dir="$status_dir/$screen_name"
|
|
||||||
local use_screen=$(trueorfalse True $USE_SCREEN)
|
|
||||||
local pid="$service_dir/$service.pid"
|
|
||||||
|
|
||||||
if is_service_enabled $1; then
|
|
||||||
# Append the service to the screen rc file
|
|
||||||
screen_rc "$service" "$cmd"
|
|
||||||
|
|
||||||
if [[ "$use_screen" = "True" ]]; then
|
|
||||||
screen -S $screen_name -X screen -t $service
|
|
||||||
|
|
||||||
# this retry loop brought to you by slow compute nodes, screen raciness,
|
|
||||||
# and frustration in upgrading.
|
|
||||||
local screen_tries=0
|
|
||||||
while [ "$screen_tries" -lt 10 ]; do
|
|
||||||
_start_in_screen "$service" "$cmd"
|
|
||||||
if _is_running_in_screen $service; then
|
|
||||||
screen_tries=10
|
|
||||||
else
|
|
||||||
screen_tries=$[screen_tries + 1]
|
|
||||||
echo "Failed to start service after $screen_tries attempt(s), retrying"
|
|
||||||
if [[ "$screen_tries" -eq 10 ]]; then
|
|
||||||
echo "Too many retries, giving up"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
else
|
else
|
||||||
# Spawn directly without screen
|
# Spawn directly without screen
|
||||||
run_process "$service" "$cmd" >$pid
|
run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user