Automate host configuration for functional testing
This change adds a new script, configure_for_func_testing.sh, that automates configuration of a host to support functional testing. The script's functionality is consumed by a refactored version of gate_hook.sh, and both minimizes runtime and removes the previous dependency on the devstack-gate repo. Additionally, the dsvm-functional tox env is no longer dependent on devstack to deploy neutron's rootwrap configuration system-wide. Rootwrap configuration is now deployed to the target tox venv on each tox invocation. Change-Id: Iaf43be458bdf3c4535f95ee5a3a3b47a744020a0
This commit is contained in:
Maru Newby
parent
c49c87d1e4
commit
540e4d791f
18
TESTING.rst
18
TESTING.rst
@ -119,11 +119,25 @@ specific-system dependencies::
|
||||
|
||||
tox -e functional
|
||||
|
||||
To run all the functional tests in an environment that has been configured
|
||||
by devstack to support sudo and system-specific dependencies::
|
||||
To run all the functional tests, including those requiring sudo
|
||||
privileges and system-specific dependencies, the procedure defined by
|
||||
tools/configure_for_func_testing.sh should be followed.
|
||||
|
||||
IMPORTANT: configure_for_func_testing.sh relies on devstack to perform
|
||||
extensive modification to the underlying host. Execution of the
|
||||
script requires sudo privileges and it is recommended that the
|
||||
following commands be invoked only on a clean and disposeable VM. A
|
||||
VM that has had devstack previously installed on it is also fine. ::
|
||||
|
||||
git clone https://git.openstack.org/openstack-dev/devstack ../devstack
|
||||
./tools/configure_for_func_testing.sh ../devstack -i
|
||||
tox -e dsvm-functional
|
||||
|
||||
The '-i' option is optional and instructs the script to use devstack
|
||||
to install and configure all of Neutron's package dependencies. It is
|
||||
not necessary to provide this option if devstack has already been used
|
||||
to deploy Neutron to the target host.
|
||||
|
||||
For more information on the standard Tox-based test infrastructure used by
|
||||
OpenStack and how to do some common test/debugging procedures with Testr,
|
||||
see this wiki page:
|
||||
|
||||
@ -4,13 +4,6 @@
|
||||
# This file should be owned by (and only-writeable by) the root user
|
||||
|
||||
[Filters]
|
||||
# '$BASE_PATH' is intended to be replaced with the expected tox path
|
||||
# (e.g. /opt/stack/new/neutron/.tox/dsvm-functional) by the neutron
|
||||
# functional jenkins job. This ensures that tests can kill the
|
||||
# processes that they launch with their containing tox environment's
|
||||
# python.
|
||||
kill_tox_python: KillFilter, root, $BASE_PATH/bin/python, -9
|
||||
|
||||
# enable ping from namespace
|
||||
ping_filter: CommandFilter, ping, root
|
||||
|
||||
@ -1,58 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
set -ex
|
||||
|
||||
|
||||
venv=${1:-"dsvm-functional"}
|
||||
|
||||
CONTRIB_DIR="$BASE/new/neutron/neutron/tests/contrib"
|
||||
|
||||
$BASE/new/devstack-gate/devstack-vm-gate.sh
|
||||
|
||||
if [ "$venv" == "dsvm-functional" ]
|
||||
then
|
||||
# Add a rootwrap filter to support test-only
|
||||
# configuration (e.g. a KillFilter for processes that
|
||||
# use the python installed in a tox env).
|
||||
FUNC_FILTER=$CONTRIB_DIR/filters.template
|
||||
sed -e "s+\$BASE_PATH+$BASE/new/neutron/.tox/dsvm-functional+" \
|
||||
$FUNC_FILTER | sudo tee /etc/neutron/rootwrap.d/functional.filters > /dev/null
|
||||
# The following need to be set before sourcing
|
||||
# configure_for_func_testing.
|
||||
GATE_DEST=$BASE/new
|
||||
GATE_STACK_USER=stack
|
||||
NEUTRON_PATH=$GATE_DEST/neutron
|
||||
DEVSTACK_PATH=$GATE_DEST/devstack
|
||||
IS_GATE=True
|
||||
|
||||
# Use devstack functions to install mysql and psql servers
|
||||
TOP_DIR=$BASE/new/devstack
|
||||
source $TOP_DIR/functions
|
||||
source $TOP_DIR/lib/config
|
||||
source $TOP_DIR/stackrc
|
||||
source $TOP_DIR/lib/database
|
||||
source $TOP_DIR/localrc
|
||||
source $NEUTRON_PATH/tools/configure_for_func_testing.sh
|
||||
|
||||
disable_service postgresql
|
||||
enable_service mysql
|
||||
initialize_database_backends
|
||||
install_database
|
||||
# Make the workspace owned by the stack user
|
||||
sudo chown -R $STACK_USER:$STACK_USER $BASE
|
||||
|
||||
disable_service mysql
|
||||
enable_service postgresql
|
||||
initialize_database_backends
|
||||
install_database
|
||||
|
||||
# Set up the 'openstack_citest' user and database in each backend
|
||||
tmp_dir=`mktemp -d`
|
||||
|
||||
cat << EOF > $tmp_dir/mysql.sql
|
||||
CREATE DATABASE openstack_citest;
|
||||
CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest';
|
||||
CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest';
|
||||
FLUSH PRIVILEGES;
|
||||
EOF
|
||||
/usr/bin/mysql -u root < $tmp_dir/mysql.sql
|
||||
|
||||
cat << EOF > $tmp_dir/postgresql.sql
|
||||
CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest';
|
||||
CREATE DATABASE openstack_citest WITH OWNER openstack_citest;
|
||||
EOF
|
||||
# User/group postgres needs to be given access to tmp_dir
|
||||
setfacl -m g:postgres:rwx $tmp_dir
|
||||
sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql
|
||||
configure_host_for_func_testing $STACK_USER
|
||||
elif [ "$venv" == "api" ]
|
||||
then
|
||||
$BASE/new/devstack-gate/devstack-vm-gate.sh
|
||||
fi
|
||||
|
||||
249
tools/configure_for_func_testing.sh
Executable file
249
tools/configure_for_func_testing.sh
Executable file
@ -0,0 +1,249 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
set -e
|
||||
|
||||
|
||||
# Control variable used to determine whether to execute this script
|
||||
# directly or allow the gate_hook to import.
|
||||
IS_GATE=${IS_GATE:-False}
|
||||
|
||||
|
||||
if [[ "$IS_GATE" != "True" ]] && [[ "$#" -lt 1 ]]; then
|
||||
>&2 echo "Usage: $0 /path/to/devstack [-i]
|
||||
Configure a host to run Neutron's functional test suite.
|
||||
|
||||
-i Install Neutron's package dependencies. By default, it is assumed
|
||||
that devstack has already been used to deploy neutron to the
|
||||
target host and that package dependencies need not be installed.
|
||||
|
||||
Warning: This script relies on devstack to perform extensive
|
||||
modification to the underlying host. It is recommended that it be
|
||||
invoked only on a throw-away VM."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# Skip the first argument
|
||||
OPTIND=2
|
||||
while getopts ":i" opt; do
|
||||
case $opt in
|
||||
i)
|
||||
INSTALL_BASE_DEPENDENCIES=True
|
||||
;;
|
||||
esac
|
||||
|
||||
done
|
||||
|
||||
# Default to environment variables to permit the gate_hook to override
|
||||
# when sourcing.
|
||||
DEVSTACK_PATH=${DEVSTACK_PATH:-$1}
|
||||
# The gate should automatically install dependencies.
|
||||
INSTALL_BASE_DEPENDENCIES=${INSTALL_BASE_DEPENDENCIES:-$IS_GATE}
|
||||
|
||||
|
||||
if [ ! -f "$DEVSTACK_PATH/stack.sh" ]; then
|
||||
>&2 echo "Unable to find devstack at '$DEVSTACK_PATH'. Please verify that the specified path points to a valid devstack repo."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
set -x
|
||||
|
||||
|
||||
function _init {
|
||||
NEUTRON_PATH=${NEUTRON_PATH:-$(cd $(dirname "$0")/.. && pwd)}
|
||||
|
||||
# Subsequently-called devstack functions depend on the following variables.
|
||||
HOST_IP=127.0.0.1
|
||||
FILES=$DEVSTACK_PATH/files
|
||||
TOP_DIR=$DEVSTACK_PATH
|
||||
|
||||
source $DEVSTACK_PATH/stackrc
|
||||
|
||||
# Allow the gate to override values set by stackrc.
|
||||
DEST=${GATE_DEST:-$DEST}
|
||||
STACK_USER=${GATE_STACK_USER:-$STACK_USER}
|
||||
}
|
||||
|
||||
|
||||
function _install_base_deps {
|
||||
echo_summary "Installing base dependencies"
|
||||
|
||||
INSTALL_TESTONLY_PACKAGES=True
|
||||
PACKAGES=$(get_packages general neutron,q-agt,q-l3)
|
||||
# Do not install 'python-' prefixed packages other than
|
||||
# python-dev*. Neutron's functional testing relies on deployment
|
||||
# to a tox env so there is no point in installing python
|
||||
# dependencies system-wide.
|
||||
PACKAGES=$(echo $PACKAGES | perl -pe 's|python-(?!dev)[^ ]*||g')
|
||||
install_package $PACKAGES
|
||||
}
|
||||
|
||||
|
||||
function _install_rpc_backend {
|
||||
echo_summary "Installing rabbitmq"
|
||||
|
||||
RABBIT_USERID=${RABBIT_USERID:-stackrabbit}
|
||||
RABBIT_HOST=${RABBIT_HOST:-$SERVICE_HOST}
|
||||
RABBIT_PASSWORD=${RABBIT_HOST:-secretrabbit}
|
||||
|
||||
source $DEVSTACK_PATH/lib/rpc_backend
|
||||
|
||||
enable_service rabbit
|
||||
install_rpc_backend
|
||||
restart_rpc_backend
|
||||
}
|
||||
|
||||
|
||||
function _install_databases {
|
||||
echo_summary "Installing databases"
|
||||
|
||||
# Avoid attempting to configure the db if it appears to already
|
||||
# have run. The setup as currently defined is not idempotent.
|
||||
if mysql openstack_citest > /dev/null 2>&1 < /dev/null; then
|
||||
echo_summary "DB config appears to be complete, skipping."
|
||||
return 0
|
||||
fi
|
||||
|
||||
MYSQL_PASSWORD=${MYSQL_PASSWORD:-secretmysql}
|
||||
DATABASE_PASSWORD=${DATABASE_PASSWORD:-secretdatabase}
|
||||
|
||||
source $DEVSTACK_PATH/lib/database
|
||||
|
||||
disable_service postgresql
|
||||
enable_service mysql
|
||||
initialize_database_backends
|
||||
install_database
|
||||
configure_database_mysql
|
||||
|
||||
disable_service mysql
|
||||
enable_service postgresql
|
||||
initialize_database_backends
|
||||
install_database
|
||||
configure_database_postgresql
|
||||
|
||||
# Set up the 'openstack_citest' user and database in each backend
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap "rm -rf $tmp_dir" EXIT
|
||||
|
||||
cat << EOF > $tmp_dir/mysql.sql
|
||||
CREATE DATABASE openstack_citest;
|
||||
CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest';
|
||||
CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest';
|
||||
FLUSH PRIVILEGES;
|
||||
EOF
|
||||
/usr/bin/mysql -u root < $tmp_dir/mysql.sql
|
||||
|
||||
cat << EOF > $tmp_dir/postgresql.sql
|
||||
CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest';
|
||||
CREATE DATABASE openstack_citest WITH OWNER openstack_citest;
|
||||
EOF
|
||||
|
||||
# User/group postgres needs to be given access to tmp_dir
|
||||
setfacl -m g:postgres:rwx $tmp_dir
|
||||
sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql
|
||||
}
|
||||
|
||||
|
||||
function _install_agent_deps {
|
||||
echo_summary "Installing agent dependencies"
|
||||
|
||||
source $DEVSTACK_PATH/lib/neutron
|
||||
|
||||
ENABLED_SERVICES=q-agt,q-dhcp,q-l3
|
||||
install_neutron_agent_packages
|
||||
}
|
||||
|
||||
|
||||
# Set up the rootwrap sudoers for neutron to target the rootwrap
|
||||
# configuration deployed in the venv.
|
||||
function _install_rootwrap_sudoers {
|
||||
echo_summary "Installing rootwrap sudoers file"
|
||||
|
||||
VENV_NAME=${venv:-dsvm-functional}
|
||||
VENV_PATH=$NEUTRON_PATH/.tox/$VENV_NAME
|
||||
ROOTWRAP_SUDOER_CMD="$VENV_PATH/bin/neutron-rootwrap $VENV_PATH/etc/neutron/rootwrap.conf *"
|
||||
TEMPFILE=$(mktemp)
|
||||
cat << EOF > $TEMPFILE
|
||||
# A bug in oslo.rootwrap [1] prevents commands executed with 'ip netns
|
||||
# exec' from being automatically qualified with a prefix from
|
||||
# rootwrap's configured exec_dirs. To work around this problem, add
|
||||
# the venv bin path to a user-specific secure_path.
|
||||
#
|
||||
# While it might seem preferable to set a command-specific
|
||||
# secure_path, this would only ensure the correct path for 'ip netns
|
||||
# exec' and the command targeted for execution in the namespace would
|
||||
# not inherit the path.
|
||||
#
|
||||
# 1: https://bugs.launchpad.net/oslo.rootwrap/+bug/1417331
|
||||
#
|
||||
Defaults:$STACK_USER secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$VENV_PATH/bin"
|
||||
$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD
|
||||
EOF
|
||||
chmod 0440 $TEMPFILE
|
||||
sudo chown root:root $TEMPFILE
|
||||
# Name the functional testing rootwrap to ensure that it will be
|
||||
# loaded after the devstack rootwrap (50_stack_sh if present) so
|
||||
# that the functional testing secure_path (a superset of what
|
||||
# devstack expects) will not be overwritten.
|
||||
sudo mv $TEMPFILE /etc/sudoers.d/60-neutron-func-test-rootwrap
|
||||
}
|
||||
|
||||
|
||||
function _install_post_devstack {
|
||||
echo_summary "Performing post-devstack installation"
|
||||
|
||||
_install_databases
|
||||
_install_rootwrap_sudoers
|
||||
|
||||
# Installing python-openvswitch from packages is a stop-gap while
|
||||
# python-openvswitch remains unavailable from pypi. This also
|
||||
# requires that sitepackages=True be set in tox.ini to allow the
|
||||
# venv to use the installed package. Once python-openvswitch
|
||||
# becomes available on pypi, this will no longer be required.
|
||||
#
|
||||
# NOTE: the package name 'python-openvswitch' is common across
|
||||
# supported distros.
|
||||
install_package python-openvswitch
|
||||
|
||||
# Configure ovs-vsctl to be reachable via the standard ovsdb port.
|
||||
sudo ovs-vsctl set-manager ptcp:6640:127.0.0.1
|
||||
}
|
||||
|
||||
|
||||
function configure_host_for_func_testing {
|
||||
echo_summary "Configuring host for functional testing"
|
||||
|
||||
if [[ "$INSTALL_BASE_DEPENDENCIES" == "True" ]]; then
|
||||
# Installing of the following can be achieved via devstack by
|
||||
# installing neutron, so their installation is conditional to
|
||||
# minimize the work to do on a devstack-configured host.
|
||||
_install_base_deps
|
||||
_install_agent_deps
|
||||
_install_rpc_backend
|
||||
fi
|
||||
_install_post_devstack
|
||||
}
|
||||
|
||||
|
||||
_init
|
||||
|
||||
|
||||
if [[ "$IS_GATE" != "True" ]]; then
|
||||
configure_host_for_func_testing
|
||||
fi
|
||||
56
tools/deploy_rootwrap.sh
Executable file
56
tools/deploy_rootwrap.sh
Executable file
@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eu
|
||||
|
||||
if [ "$#" -ne 3 ]; then
|
||||
>&2 echo "Usage: $0 /path/to/neutron /path/to/target/etc /path/to/target/bin
|
||||
Deploy Neutron's rootwrap configuration.
|
||||
|
||||
Warning: Any existing rootwrap files at the specified etc path will be
|
||||
removed by this script.
|
||||
|
||||
Optional: set OS_SUDO_TESTING=1 to deploy the filters required by
|
||||
Neutron's functional testing suite."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OS_SUDO_TESTING=${OS_SUDO_TESTING:-0}
|
||||
|
||||
neutron_path=$1
|
||||
target_etc_path=$2
|
||||
target_bin_path=$3
|
||||
|
||||
src_conf_path=${neutron_path}/etc
|
||||
src_conf=${src_conf_path}/rootwrap.conf
|
||||
src_rootwrap_path=${src_conf_path}/neutron/rootwrap.d
|
||||
|
||||
dst_conf_path=${target_etc_path}/neutron
|
||||
dst_conf=${dst_conf_path}/rootwrap.conf
|
||||
dst_rootwrap_path=${dst_conf_path}/rootwrap.d
|
||||
|
||||
if [[ -d "$dst_rootwrap_path" ]]; then
|
||||
rm -rf ${dst_rootwrap_path}
|
||||
fi
|
||||
mkdir -p -m 755 ${dst_rootwrap_path}
|
||||
|
||||
cp -p ${src_rootwrap_path}/* ${dst_rootwrap_path}/
|
||||
cp -p ${src_conf} ${dst_conf}
|
||||
sed -i "s:^filters_path=.*$:filters_path=${dst_rootwrap_path}:" ${dst_conf}
|
||||
sed -i "s:^\(exec_dirs=.*\)$:\1,${target_bin_path}:" ${dst_conf}
|
||||
|
||||
if [[ "$OS_SUDO_TESTING" = "1" ]]; then
|
||||
cp -p ${neutron_path}/neutron/tests/contrib/functional-testing.filters \
|
||||
${dst_rootwrap_path}/
|
||||
fi
|
||||
@ -32,7 +32,7 @@ check_opinionated_shell () {
|
||||
# If you cannot avoid the use of bash, please change the EXPECTED var below.
|
||||
OBSERVED=$(grep -E '^([[:space:]]*[^#[:space:]]|#!).*bash' \
|
||||
tox.ini tools/* | wc -l)
|
||||
EXPECTED=4
|
||||
EXPECTED=6
|
||||
if [ ${EXPECTED} -ne ${OBSERVED} ]; then
|
||||
echo "Bash usage has been detected!" >>$FAILURES
|
||||
fi
|
||||
|
||||
3
tox.ini
3
tox.ini
@ -14,6 +14,7 @@ deps = -r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
whitelist_externals = sh
|
||||
commands =
|
||||
dsvm-functional: {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envbindir}
|
||||
sh tools/pretty_tox.sh '{posargs}'
|
||||
# there is also secret magic in pretty_tox.sh which lets you run in a fail only
|
||||
# mode. To do this define the TRACE_FAILONLY environmental variable.
|
||||
@ -37,7 +38,7 @@ deps =
|
||||
[testenv:dsvm-functional]
|
||||
setenv = OS_TEST_PATH=./neutron/tests/functional
|
||||
OS_SUDO_TESTING=1
|
||||
OS_ROOTWRAP_CMD=sudo /usr/local/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
|
||||
OS_ROOTWRAP_CMD=sudo {envbindir}/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
|
||||
OS_FAIL_ON_MISSING_DEPS=1
|
||||
OS_TEST_TIMEOUT=90
|
||||
sitepackages=True
|
||||
|
||||
Loading…
Reference in New Issue
Block a user