4c9c059883
We fixed the tempest venv constraints setting for tempest test run on base branch but forgot to do the same for target branch test run. It cause issue when base and target branch have different Tempest and constraints to use (for example stable/wallaby use old tmepest and stable/wallaby constraints but stable/xena use tempest master and master constraints), in such cases we need to set proper constraints defined in devstack and then run tempest. We do not see this issue as none of the job run tempest on target except heat (https://review.opendev.org/c/openstack/heat/+/872055/5). I reproduced the issue in grenade stable/xena by enaling the tempest run on target: - https://review.opendev.org/c/openstack/grenade/+/878247/1 - https://zuul.opendev.org/t/openstack/build/1b503d359717459c9c77010608068e27/log/controller/logs/grenade.sh_log.txt#17184 Closes-Bug: 2003993 Change-Id: I05fcdb5045a33997bd1a4c340a16458d88399f5f
391 lines
13 KiB
Bash
Executable File
391 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# ``grenade.sh`` is an OpenStack upgrade test harness to exercise the
|
|
# OpenStack upgrade process. It uses DevStack to perform the initial
|
|
# OpenStack install.
|
|
|
|
# Grenade assumes it is running on the system that will be hosting the
|
|
# upgrade processes
|
|
|
|
# ``grenade.sh [-b] [-t] [-s stop-label] [-q]``
|
|
#
|
|
# ``-b`` Run only the base part
|
|
# ``-t`` Run only the target part (assumes a base run is in place)
|
|
# ``-q`` Quiet mode
|
|
# ``-s stop-label`` is the name of the step after which the script will stop.
|
|
# This is useful for debugging upgrades.
|
|
|
|
|
|
|
|
# ``GRENADE_DIR`` is set once by the top level grenade.sh and exported
|
|
# so that all subsequent scripts can find their way back to the
|
|
# grenade root directory. No other scripts should set this variable
|
|
export GRENADE_DIR=$(cd $(dirname "$0") && pwd)
|
|
|
|
# Source the bootstrapping facilities
|
|
#
|
|
# Grenade attempts to reuse as much content from devstack on the
|
|
# target side as possible, but we need enough of our own code to get
|
|
# there.
|
|
#
|
|
# ``grenaderc`` is a set of X=Y declarations that don't need *any* of
|
|
# the devstack functions to work
|
|
#
|
|
# ``inc/bootstrap`` is the most minimal amount of functions that
|
|
# grenade needs to get going. This includes things like echo
|
|
# functions, trueorfalse, and the functions related to git cloning, so
|
|
# that we can get our devstack trees.
|
|
export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.4.0}
|
|
source $GRENADE_DIR/grenaderc
|
|
source $GRENADE_DIR/inc/bootstrap
|
|
|
|
while getopts bqs:t c; do
|
|
case $c in
|
|
b)
|
|
RUN_TARGET=False
|
|
;;
|
|
q)
|
|
VERBOSE=False
|
|
;;
|
|
s)
|
|
STOP=$2
|
|
;;
|
|
t)
|
|
RUN_BASE=False
|
|
;;
|
|
esac
|
|
done
|
|
shift `expr $OPTIND - 1`
|
|
|
|
|
|
# Create all the base directory structures needed for the rest of the
|
|
# environment to run.
|
|
#
|
|
# This will give you an STACK_ROOT tree that you expect, and that your
|
|
# normal user owns for follow on activities.
|
|
sudo mkdir -p $BASE_RELEASE_DIR $TARGET_RELEASE_DIR
|
|
sudo chown -R `whoami` $STACK_ROOT
|
|
|
|
# Logging
|
|
# =======
|
|
|
|
# TODO(sdague): should this extract into ``inc/bootstrap``?
|
|
#
|
|
# Set up logging
|
|
# Set ``LOGFILE`` to turn on logging
|
|
# Append '.xxxxxxxx' to the given name to maintain history
|
|
# where 'xxxxxxxx' is a representation of the date the file was created
|
|
if [[ -n "$LOGFILE" ]]; then
|
|
LOGDAYS=${LOGDAYS:-7}
|
|
TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"}
|
|
CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT")
|
|
|
|
# First clean up old log files. Use the user-specified ``LOGFILE``
|
|
# as the template to search for, appending '.*' to match the date
|
|
# we added on earlier runs.
|
|
LOGDIR=$(dirname "$LOGFILE")
|
|
LOGNAME=$(basename "$LOGFILE")
|
|
echo "Creating $LOGDIR...."
|
|
|
|
sudo mkdir -p $LOGDIR
|
|
sudo chown -R `whoami` $LOGDIR
|
|
find $LOGDIR -maxdepth 1 -name $LOGNAME.\* -mtime +$LOGDAYS -exec rm {} \;
|
|
LOGFILE=$LOGFILE.${CURRENT_LOG_TIME}
|
|
SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary
|
|
|
|
# Redirect output according to config
|
|
# Copy stdout to fd 3
|
|
exec 3>&1
|
|
if [[ "$VERBOSE" == "True" ]]; then
|
|
echo "Running in verbose mode:"
|
|
echo " Full logs found at => ${LOGFILE}"
|
|
echo " Summary logs at => ${SUMFILE}"
|
|
# Redirect stdout/stderr to tee to write the log file
|
|
exec 1> >( ./tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1
|
|
# Set up a second fd for output
|
|
exec 6> >( ./tools/outfilter.py -o "${SUMFILE}" )
|
|
else
|
|
echo "Running in summary mode:"
|
|
echo " Full logs found at => ${LOGFILE}"
|
|
echo " Summary logs at => ${SUMFILE}"
|
|
# Set fd 1 and 2 to primary logfile
|
|
exec 1> >( ./tools/outfilter.py -o "${LOGFILE}") 2>&1
|
|
# Set fd 6 to summary logfile and stdout
|
|
exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3)
|
|
fi
|
|
|
|
echo_summary "grenade.sh log $LOGFILE"
|
|
# Specified logfile name always links to the most recent log
|
|
ln -sf $LOGFILE $LOGDIR/$LOGNAME
|
|
ln -sf $SUMFILE $LOGDIR/$LOGNAME.summary
|
|
else
|
|
# Set up output redirection without log files
|
|
# Copy stdout to fd 3
|
|
exec 3>&1
|
|
if [[ "$VERBOSE" != "yes" ]]; then
|
|
# Throw away stdout and stderr
|
|
exec 1>/dev/null 2>&1
|
|
fi
|
|
# Always send summary fd to original stdout
|
|
exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3)
|
|
fi
|
|
|
|
# Set up logging of screen windows
|
|
# Set ``SCREEN_LOGDIR`` to turn on logging of screen windows to the
|
|
# directory specified in ``SCREEN_LOGDIR``, we will log to the file
|
|
# ``screen-$SERVICE_NAME-$TIMESTAMP.log`` in that dir and have a link
|
|
# ``screen-$SERVICE_NAME.log`` to the latest log file.
|
|
# Logs are kept for as long specified in ``LOGDAYS``.
|
|
if [[ -n "$SCREEN_LOGDIR" ]]; then
|
|
|
|
# We make sure the directory is created.
|
|
if [[ -d "$SCREEN_LOGDIR" ]]; then
|
|
# We cleanup the old logs
|
|
find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \;
|
|
else
|
|
sudo mkdir -p $SCREEN_LOGDIR
|
|
sudo chown -R `whoami` $SCREEN_LOGDIR
|
|
fi
|
|
fi
|
|
|
|
|
|
# This script exits on an error so that errors don't compound and you see
|
|
# only the first error that occurred.
|
|
set -o errexit
|
|
|
|
# Print the commands being run so that we can see the command that triggers
|
|
# an error. It is also useful for following allowing as the install occurs.
|
|
set -o xtrace
|
|
|
|
|
|
# Devstack Phase 2 initialization
|
|
# ===============================
|
|
#
|
|
# We now have enough infrastructure in grenade.sh to go and get *both*
|
|
# SOURCE and TARGET devstack trees. After which point we 'pivot' onto
|
|
# the TARGET devstack functions file, then source the rest of the
|
|
# grenade settings. This should let us run the bulk of grenade.
|
|
|
|
if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
|
|
# Get both devstack trees, so that BASE_DEVSTACK_DIR, and
|
|
# TARGET_DEVSTACK_DIR are now fully populated.
|
|
fetch_devstacks
|
|
else
|
|
devstacks_setup_environment
|
|
fi
|
|
|
|
# Source the rest of the Grenade functions. For convenience
|
|
# ``$GRENADE_DIR/functions`` implicitly sources
|
|
# ``$TARGET_DEVSTACK_DIR/functions``. So this line can't happen until
|
|
# we have the devstacks pulled down.
|
|
source $GRENADE_DIR/functions
|
|
|
|
# We now have the 'short_source' function available, so setup our PS4 variable
|
|
export PS4='+ $(short_source): '
|
|
|
|
# Many calls inside of devstack functions reference $TOP_DIR, which is
|
|
# the root of devstack. We export $TOP_DIR to all child processes here
|
|
# to be the TARGET_DEVSTACK_DIR.
|
|
#
|
|
# If you want a script to use functions off of BASE_DEVSTACK_DIR (like
|
|
# the shutdown phase) you *must* explicitly reset TOP_DIR in those
|
|
# scripts.
|
|
export TOP_DIR=$TARGET_DEVSTACK_DIR
|
|
|
|
# Install 'Base' Build of OpenStack
|
|
# =================================
|
|
|
|
source $TARGET_DEVSTACK_DIR/inc/meta-config
|
|
# Oh the complexity of bootstrapping. We need to populate enabled
|
|
# services from what devstack-gate might have set it as.
|
|
extract_localrc_section $BASE_DEVSTACK_DIR/local.conf \
|
|
$BASE_DEVSTACK_DIR/localrc \
|
|
$BASE_DEVSTACK_DIR/.localrc.auto
|
|
|
|
# Collect the ENABLED_SERVICES from the base directory, this is what
|
|
# we are starting with.
|
|
ENABLED_SERVICES=$(set +o xtrace &&
|
|
source $BASE_DEVSTACK_DIR/stackrc &&
|
|
echo $ENABLED_SERVICES)
|
|
|
|
# Fetch all the grenade plugins which were registered in ``pluginrc``
|
|
# via the ``enable_grenade_plugin`` stanza. This must be done before
|
|
# settings are loaded, but has to be this late to give access to all
|
|
# the devstack functions.
|
|
fetch_grenade_plugins
|
|
|
|
# Load the ``settings`` files for all the in tree ``projects/``. This
|
|
# registers all the projects in order that we're going to be upgrading
|
|
# when the time is right.
|
|
load_settings
|
|
|
|
if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
|
|
# Nova should use singleconductor as Grenade doesn't
|
|
# setup multi-cell rabbit for now
|
|
devstack_localrc base "CELLSV2_SETUP=singleconductor"
|
|
devstack_localrc target "CELLSV2_SETUP=singleconductor"
|
|
fi
|
|
|
|
# And ensure that we setup the target localrc.auto, because stack.sh
|
|
# isn't run there. This has to be run after load_settings because
|
|
# plugins might change the service list during this phase.
|
|
|
|
extract_localrc_section $TARGET_DEVSTACK_DIR/local.conf \
|
|
$TARGET_DEVSTACK_DIR/localrc \
|
|
$TARGET_DEVSTACK_DIR/.localrc.auto
|
|
|
|
# Run the base install of the environment
|
|
if [[ "$RUN_BASE" == "True" ]]; then
|
|
|
|
# Initialize grenade_db local storage, used for resource tracking
|
|
init_grenade_db
|
|
|
|
if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
|
|
echo_summary "Running base stack.sh"
|
|
cd $BASE_DEVSTACK_DIR
|
|
GIT_BASE=$GIT_BASE ./stack.sh
|
|
stop $STOP stack.sh 10
|
|
|
|
echo_summary "Running post-stack.sh"
|
|
if [[ -e $GRENADE_DIR/post-stack.sh ]]; then
|
|
cd $GRENADE_DIR
|
|
|
|
# By the time we get here the sub nodes are already setup with localrc files
|
|
# as those are transferred in devstack-gate even before grenade.sh is called
|
|
# We hack the ./post-stack.sh to inject what we need. if we don't set
|
|
# CELLSV2_SETUP, the default devstack assumes "superconductor" and fails.
|
|
export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor"
|
|
sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh
|
|
cat ./post-stack.sh
|
|
|
|
./post-stack.sh
|
|
stop $STOP post-stack.sh 15
|
|
echo_summary "Completed post-stack.sh"
|
|
fi
|
|
fi
|
|
|
|
# Cache downloaded instances
|
|
# --------------------------
|
|
|
|
echo_summary "Caching downloaded images"
|
|
mkdir -p $BASE_RELEASE_DIR/images
|
|
echo "Images: $IMAGE_URLS"
|
|
for image_url in ${IMAGE_URLS//,/ }; do
|
|
IMAGE_FNAME=`basename "$image_url"`
|
|
if [[ -r $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME ]]; then
|
|
rsync -a $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME $BASE_RELEASE_DIR/images
|
|
fi
|
|
done
|
|
# NOTE(sileht): If glance is not enabled the directory cannot exists.
|
|
if [[ -d $BASE_DEVSTACK_DIR/files/images ]] ; then
|
|
rsync -a $BASE_DEVSTACK_DIR/files/images/ $BASE_RELEASE_DIR/images
|
|
fi
|
|
stop $STOP image-cache 20
|
|
|
|
# Operation
|
|
# ---------
|
|
|
|
# Validate the install
|
|
if [[ "$BASE_RUN_SMOKE" == "True" ]]; then
|
|
echo_summary "Running base smoke test"
|
|
run_tempest $BASE_RELEASE_DIR
|
|
if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then
|
|
# once we are done, copy our created artifacts to the target
|
|
if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then
|
|
rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/
|
|
if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then
|
|
rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/
|
|
elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then
|
|
rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
stop $STOP base-smoke 110
|
|
|
|
# Early Create resources, used largely for network setup
|
|
resources early_create
|
|
|
|
# Create resources
|
|
resources create
|
|
|
|
# Verify the resources were created
|
|
resources verify pre-upgrade
|
|
|
|
# Save some stuff before we shut that whole thing down
|
|
echo_summary "Saving current state information"
|
|
$GRENADE_DIR/save-state
|
|
stop $STOP save-state 130
|
|
|
|
# Shut down running code
|
|
echo_summary "Shutting down all services on base devstack..."
|
|
shutdown_services
|
|
|
|
# Verify the resources still exist after the shutdown
|
|
resources verify_noapi pre-upgrade
|
|
fi
|
|
|
|
|
|
# Upgrades
|
|
# ========
|
|
|
|
if [[ "$RUN_TARGET" == "True" ]]; then
|
|
# Clone all devstack plugins on the new side, because we're not
|
|
# running a full stack.sh
|
|
fetch_plugins
|
|
|
|
# Get target devstack tree ready for services to be run from it,
|
|
# including trying to reuse any existing files we pulled during
|
|
# the base run.
|
|
echo_summary "Preparing the target devstack environment"
|
|
$GRENADE_DIR/prep-target
|
|
|
|
# upgrade all the projects in order
|
|
echo "Upgrade projects: $UPGRADE_PROJECTS"
|
|
for project in $UPGRADE_PROJECTS; do
|
|
echo "Upgrading project: $project"
|
|
upgrade_service $project
|
|
done
|
|
|
|
# Upgrade Tempest
|
|
if [[ "$ENABLE_TEMPEST" == "True" ]]; then
|
|
echo_summary "Running upgrade-tempest"
|
|
$GRENADE_DIR/upgrade-tempest || die $LINENO "Failure in upgrade-tempest"
|
|
stop $STOP upgrade-tempest 290
|
|
fi
|
|
|
|
# Upgrade Tests
|
|
# =============
|
|
|
|
# Verify the resources still exist after the upgrade
|
|
resources verify post-upgrade
|
|
|
|
# Validate the upgrade
|
|
# This is used for testing grenade locally, and should not be used in the
|
|
# gate. Instead, grenade.sh runs smoke tests on the old cloud above, but
|
|
# smoke tests are run on the upgraded cloud by the gate script after
|
|
# grenade.sh has finished. If this option is enabled in the gate, tempest
|
|
# will be run twice against the new cloud.
|
|
if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then
|
|
echo_summary "Running tempest smoke tests"
|
|
run_tempest $TARGET_RELEASE_DIR
|
|
stop $STOP run-smoke 330
|
|
fi
|
|
|
|
# Save databases
|
|
# --------------
|
|
save_data $TARGET_RELEASE $TARGET_DEVSTACK_DIR
|
|
|
|
# Cleanup the resources
|
|
resources destroy
|
|
|
|
fi
|
|
|
|
|
|
# Fin
|
|
# ===
|
|
|
|
echo_summary "Grenade has completed the pre-programmed upgrade scripts."
|
|
# Indicate how long this took to run (bash maintained variable ``SECONDS``)
|
|
echo_summary "grenade.sh completed in $SECONDS seconds."
|