5c2702e7a9
This aims to ensure that collections which are managed by zuul will get installed and used during testing. This is especially important since our deliverables are being installed as collections. As a base we take code from the integrated repo and simplify it a bit by removing user defined collections. Change-Id: I92b6b3edd5d781b3ce9292b52773b0a854f910c7
260 lines
9.8 KiB
Bash
Executable File
260 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2016, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# WARNING:
|
|
# This file is use by all OpenStack-Ansible roles for testing purposes.
|
|
# Any changes here will affect all OpenStack-Ansible role repositories
|
|
# with immediate effect.
|
|
|
|
# PURPOSE:
|
|
# This script prepares the host with all the required Ansible
|
|
# roles and plugins to execute the test playbook.
|
|
|
|
## Shell Opts ----------------------------------------------------------------
|
|
|
|
set -e
|
|
set -x
|
|
|
|
## Vars ----------------------------------------------------------------------
|
|
|
|
export TESTING_HOME=${TESTING_HOME:-$HOME}
|
|
export WORKING_DIR=${WORKING_DIR:-$(pwd)}
|
|
export ROLE_NAME=${ROLE_NAME:-''}
|
|
export OSA_PROJECT_NAME="$(sed -n 's|^project=openstack/\(.*\).git$|\1|p' $(pwd)/.gitreview)"
|
|
export ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY:-$WORKING_DIR/tests/inventory}
|
|
export ANSIBLE_ROLE_REQUIREMENTS_PATH=${ANSIBLE_ROLE_REQUIREMENTS_PATH:-$WORKING_DIR/tests/ansible-role-requirements.yml}
|
|
export ANSIBLE_EXTRA_ROLE_DIRS=${ANSIBLE_EXTRA_ROLE_DIRS:-''}
|
|
|
|
export ANSIBLE_CFG_PATH="${TESTING_HOME}/.ansible.cfg"
|
|
export ANSIBLE_LOG_DIR="${TESTING_HOME}/.ansible/logs"
|
|
export ANSIBLE_NOCOLOR=1
|
|
export ANSIBLE_PLUGIN_DIR="${TESTING_HOME}/.ansible/plugins"
|
|
export ANSIBLE_ROLE_DEP_DIR="${TESTING_HOME}/.ansible/roles"
|
|
export ANSIBLE_ROLE_DIR="${TESTING_HOME}/.ansible/testing-role"
|
|
export COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
|
|
|
|
# The place where zuul clones dependent repositories to
|
|
ZUUL_PLUGINS_CLONE_LOCATION="/home/zuul/src/opendev.org/openstack/openstack-ansible-plugins"
|
|
|
|
# Use .gitreview as the key to determine the appropriate
|
|
# branch to clone for tests.
|
|
export TESTING_BRANCH=${TESTING_BRANCH:-`awk -F'=' '/defaultbranch/ {print $2}' "${WORKING_DIR}/.gitreview"`}
|
|
if [[ "${TESTING_BRANCH}" == "" ]]; then
|
|
export TESTING_BRANCH="master"
|
|
fi
|
|
|
|
export SERVICES_BRANCH=${SERVICES_BRANCH:-$TESTING_BRANCH}
|
|
|
|
# Use pip opts to add options to the pip install command.
|
|
# This can be used to tell it which index to use, etc.
|
|
export PIP_OPTS=${PIP_OPTS:-""}
|
|
|
|
echo "TESTING_HOME: ${TESTING_HOME}"
|
|
echo "TESTING_BRANCH: ${TESTING_BRANCH}"
|
|
echo "SERVICES_BRANCH: ${SERVICES_BRANCH}"
|
|
echo "WORKING_DIR: ${WORKING_DIR}"
|
|
echo "ROLE_NAME: ${ROLE_NAME}"
|
|
echo "ANSIBLE_INVENTORY: ${ANSIBLE_INVENTORY}"
|
|
|
|
# Output all the zuul parameters if they're set
|
|
if [ -z "${ZUUL_CHANGE_IDS}" ]; then
|
|
echo -e "\n### ZUUL PARAMETERS BEGIN ###\n"
|
|
printenv | grep ^ZUUL || true
|
|
echo -e "\n### ZUUL PARAMETERS END ###\n"
|
|
fi
|
|
|
|
# Toggle the reset of all data cloned from other repositories.
|
|
export TEST_RESET=${TEST_RESET:-false}
|
|
|
|
# Make sure that python is not buffering output so that the
|
|
# console output is immediate.
|
|
export PYTHONUNBUFFERED=1
|
|
|
|
## Functions -----------------------------------------------------------------
|
|
|
|
function setup_ara {
|
|
|
|
# Don't do anything if ARA has already been set up
|
|
[[ -L "${ANSIBLE_PLUGIN_DIR}/callback/ara" ]] && return 0
|
|
|
|
# Dynamically figure out the location of ARA (ex: py2 vs py3)
|
|
ara_location=$(python -m ara.setup.callback_plugins 2>/dev/null)
|
|
|
|
echo "Linking ${ara_location} to ${ANSIBLE_PLUGIN_DIR}/callback/ara"
|
|
mkdir -p "${ANSIBLE_PLUGIN_DIR}/callback/ara"
|
|
if [ -d ${ara_location} ]; then
|
|
ln -sf "${ara_location}" "${ANSIBLE_PLUGIN_DIR}/callback/ara/"
|
|
fi
|
|
|
|
}
|
|
|
|
## Main ----------------------------------------------------------------------
|
|
|
|
# If the test reset toggle is set, destroy the existing cloned data.
|
|
if [ "${TEST_RESET}" == "true" ]; then
|
|
echo "Resetting all cloned data."
|
|
rm -f "${ANSIBLE_CFG_PATH}"
|
|
rm -rf "${ANSIBLE_LOG_DIR}"
|
|
rm -rf "${ANSIBLE_PLUGIN_DIR}"
|
|
rm -rf "${ANSIBLE_ROLE_DIR}"
|
|
fi
|
|
|
|
# Create the directory which will hold all Ansible logs
|
|
mkdir -p "${ANSIBLE_LOG_DIR}"
|
|
|
|
if [[ ! -d "${ANSIBLE_PLUGIN_DIR}" ]]; then
|
|
# The plugins repo doesn't need a clone, we can just
|
|
# symlink it.
|
|
if [[ "${OSA_PROJECT_NAME}" == "openstack-ansible-plugins" ]]; then
|
|
ln -s ${WORKING_DIR} "${ANSIBLE_PLUGIN_DIR}"
|
|
|
|
# In zuul v3 any dependent repository is placed into
|
|
# /home/zuul/src/opendev.org, so we check to see
|
|
# if there is a tests checkout there already. If so, we
|
|
# symlink that and use it.
|
|
elif [[ -d "${ZUUL_PLUGINS_CLONE_LOCATION}" ]]; then
|
|
ln -s "${ZUUL_PLUGINS_CLONE_LOCATION}" "${ANSIBLE_PLUGIN_DIR}"
|
|
|
|
# Otherwise we're clearly not in zuul or using a previously setup
|
|
# repo in some way, so just clone it from upstream.
|
|
else
|
|
git clone -b "${TESTING_BRANCH}" \
|
|
https://opendev.org/openstack/openstack-ansible-plugins \
|
|
"${ANSIBLE_PLUGIN_DIR}"
|
|
fi
|
|
fi
|
|
|
|
# Ensure we use the HTTPS/HTTP proxy with pip if it is specified
|
|
if [ -n "$HTTPS_PROXY" ]; then
|
|
PIP_OPTS+=" --proxy $HTTPS_PROXY"
|
|
elif [ -n "$HTTP_PROXY" ]; then
|
|
PIP_OPTS+=" --proxy $HTTP_PROXY"
|
|
fi
|
|
|
|
# Using tox for requirements management requires in-repo
|
|
# requirements files for all our repositories. Rather than
|
|
# do that, we make use of the tests repo to capture our
|
|
# common requirements and use this to install them.
|
|
# This reduces our review requirement rate and simplifies
|
|
# maintenance for us for the tox config. It also makes it
|
|
# usable with 'Depends-On', which is marvellous!
|
|
|
|
# If the repo has its own test-requirements file, then use
|
|
# it with the common one.
|
|
if [[ -f "${WORKING_DIR}/test-requirements.txt" ]]; then
|
|
PIP_OPTS+=" --requirement ${WORKING_DIR}/test-requirements.txt"
|
|
fi
|
|
|
|
# We add the common requirements after the in-repo requirements
|
|
# so that the in-repo ones take precedence.
|
|
PIP_OPTS+=" --requirement ${COMMON_TESTS_PATH}/test-requirements.txt"
|
|
|
|
# If the repo has a doc/requirements.txt file, add it to the
|
|
# requirements list. This is necessary for the linters test
|
|
# to be able to execute doc8.
|
|
if [[ -f "${WORKING_DIR}/doc/requirements.txt" && "${TOX_ENV_NAME}" != "functional"* ]]; then
|
|
PIP_OPTS+=" --requirement ${WORKING_DIR}/doc/requirements.txt"
|
|
fi
|
|
|
|
# We want to install ansible, but also constrain it.
|
|
# This is necessary due to ARA having ansible as a
|
|
# requirement.
|
|
PIP_OPTS+=" --requirement ${COMMON_TESTS_PATH}/test-ansible-deps.txt"
|
|
|
|
# Retrieve a copy of OpenStack's upper constraints If Depends-On is
|
|
# used, the requirements repo will be cloned, so we prefer
|
|
# local copy from zuul.
|
|
REQS_REPO_HOME="${TESTING_HOME}/src/opendev.org/openstack/requirements"
|
|
if [[ -d "${REQS_REPO_HOME}" ]]; then
|
|
cp "${REQS_REPO_HOME}/upper-constraints.txt" /tmp/upper-constraints.txt
|
|
else
|
|
curl -L "${TOX_CONSTRAINTS_FILE:-https://opendev.org/openstack/requirements/raw/${SERVICES_BRANCH}/upper-constraints.txt}" -o /tmp/upper-constraints.txt
|
|
fi
|
|
|
|
# Filter out pip/wheel from OpenStack upper constraints
|
|
# These are already constrained in OSA global-requirement-pins.txt
|
|
cp /tmp/upper-constraints.txt /tmp/upper-constraints-filtered.txt
|
|
sed -i '/pip=/d' /tmp/upper-constraints-filtered.txt
|
|
sed -i '/wheel=/d' /tmp/upper-constraints-filtered.txt
|
|
|
|
PIP_OPTS+=" --constraint /tmp/upper-constraints-filtered.txt"
|
|
|
|
source /etc/os-release || source /usr/lib/os-release
|
|
|
|
# Install ARA from PyPi
|
|
PIP_OPTS+=" ara[server]"
|
|
|
|
# Install all python packages
|
|
python -m pip install ${PIP_OPTS}
|
|
|
|
# Install all ansible collections
|
|
ANSIBLE_COLLECTION_REQUIREMENTS_PATH="${COMMON_TESTS_PATH}/test-ansible-collection-requirements.yml" \
|
|
ansible-playbook -i ${ANSIBLE_INVENTORY} ${COMMON_TESTS_PATH}/get-ansible-collection-requirements.yml -v
|
|
|
|
# Download the Ansible role repositories if they are not present on the host.
|
|
# This is ignored if there is no ansible-role-requirements file.
|
|
if [[ ! -d "${ANSIBLE_ROLE_DEP_DIR}" ]]; then
|
|
# Download the common test Ansible role repositories.
|
|
ANSIBLE_ROLE_REQUIREMENTS_PATH=${COMMON_TESTS_PATH}/test-ansible-role-requirements.yml \
|
|
ansible-playbook -i ${ANSIBLE_INVENTORY} \
|
|
${COMMON_TESTS_PATH}/get-ansible-role-requirements.yml \
|
|
-v
|
|
|
|
if [[ -f "${ANSIBLE_ROLE_REQUIREMENTS_PATH}" ]]; then
|
|
ansible-playbook -i ${ANSIBLE_INVENTORY} \
|
|
${COMMON_TESTS_PATH}/get-ansible-role-requirements.yml \
|
|
-v
|
|
if [[ ! -e "${ANSIBLE_ROLE_DEP_DIR}/plugins" ]]; then
|
|
ln -s "${ANSIBLE_PLUGIN_DIR}" "${ANSIBLE_ROLE_DEP_DIR}/plugins"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# If a role name is provided, replace the role in the roles folder with a link
|
|
# to the current folder. This ensures that the test executes with the checked
|
|
# out git repo.
|
|
if [ ! -z "${ROLE_NAME}" ]; then
|
|
echo "Linking ${ANSIBLE_ROLE_DIR}/${ROLE_NAME} to ${WORKING_DIR}"
|
|
mkdir -p "${ANSIBLE_ROLE_DIR}"
|
|
rm -rf "${ANSIBLE_ROLE_DIR}/${ROLE_NAME}"
|
|
ln -s "${WORKING_DIR}" "${ANSIBLE_ROLE_DIR}/${ROLE_NAME}"
|
|
else
|
|
echo "Skipping the role link because no role name was provided."
|
|
fi
|
|
|
|
# Ensure that the Ansible configuration file is in the right place
|
|
if [ ! -f "${ANSIBLE_CFG_PATH}" ]; then
|
|
if [ -f "${COMMON_TESTS_PATH}/test-ansible.cfg" ]; then
|
|
echo "Linking ${ANSIBLE_CFG_PATH} to ${COMMON_TESTS_PATH}/test-ansible.cfg"
|
|
ln -s "${COMMON_TESTS_PATH}/test-ansible.cfg" "${ANSIBLE_CFG_PATH}"
|
|
else
|
|
echo "Skipping the ansible.cfg link because ${COMMON_TESTS_PATH}/test-ansible.cfg is not there!"
|
|
fi
|
|
else
|
|
echo "Found ${ANSIBLE_CFG_PATH} so there's nothing more to do."
|
|
fi
|
|
|
|
# Adjust the Ansible configuration file to include the extra
|
|
# role paths if any are provided and they're not already set.
|
|
if [ ! -z "${ANSIBLE_EXTRA_ROLE_DIRS}" ]; then
|
|
if ! grep -q "roles_path.*${ANSIBLE_EXTRA_ROLE_DIRS}" "${ANSIBLE_CFG_PATH}"; then
|
|
sed -i "s|roles_path.*HOME/.ansible/roles.*|roles_path = $HOME/.ansible/roles:${ANSIBLE_ROLE_DIR}:${ANSIBLE_EXTRA_ROLE_DIRS}|" "${ANSIBLE_CFG_PATH}"
|
|
fi
|
|
fi
|
|
|
|
# Setup ARA
|
|
setup_ara
|