Implement centralised test scripts

This patch implements test scripts intended for use by all
OpenStack-Ansible role tests.

The intent is to simplify the role tox.ini configuration
and ensure that as many changes to role testing configuration
can be managed from the centralised tests repo instead of
individually in each repository.

The common tests repo target location when cloned is changed
from 'tests/playbooks' to 'tests/common' to better reflect
the content and purpose.

Change-Id: I1a93329ada24670c87588ce5a07f92c9bce4e50b

run_tests.sh

@@ -45,11 +45,9 @@ fi
 
 # run through each tox env and execute the test
 for tox_env in $(awk -F= '/envlist/ {print $2}' tox.ini | sed 's/,/ /g'); do
-  if [ "${tox_env}" != "ansible-functional" ]; then
+  if [ "${tox_env}" != "functional" ]; then
     tox -e ${tox_env}
-  elif [ "${tox_env}" == "ansible-functional" ]; then
+  elif [ "${tox_env}" == "functional" ] && [ "${FUNCTIONAL_TEST}" == "true" ]; then
-    if ${FUNCTIONAL_TEST}; then
     tox -e ${tox_env}
   fi
-  fi
 done

test-ansible-deps.txt

@@ -0,0 +1,14 @@
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# Python requirements listed here are imported by the roles via tox
+# target configuration in each role.
+
+# The Ansible version used for testing
+ansible==2.1.1
+
+# The Ansible lint version used for lint tests
+ansible-lint>=2.7.0,<3.0.0

test-ansible-env-prep.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script prepares the host with all the required Ansible
+# roles and plugins to execute the test playbook.
+
+export TESTING_HOME=${TESTING_HOME:-$HOME}
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+export ROLE_NAME=${ROLE_NAME:-''}
+
+export ANSIBLE_ROLE_DIR="${TESTING_HOME}/.ansible/roles"
+export ANSIBLE_PLUGIN_DIR="${TESTING_HOME}/.ansible/plugins"
+export ANSIBLE_CFG_PATH="${TESTING_HOME}/.ansible.cfg"
+export ANSIBLE_ROLE_REQUIREMENTS_PATH="${WORKING_DIR}/tests/ansible-role-requirements.yml"
+export COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
+
+echo "TESTING_HOME: ${TESTING_HOME}"
+echo "WORKING_DIR: ${WORKING_DIR}"
+echo "ROLE_NAME: ${ROLE_NAME}"
+
+# Toggle the reset of all data cloned from other repositories.
+export TEST_RESET=${TEST_RESET:-false}
+
+# Make sure that python is not buffering output so that the
+# console output is immediate.
+export PYTHONUNBUFFERED=1
+
+# If the test reset toggle is set, destroy the existing cloned data.
+if [ "${TEST_RESET}" == "true" ]; then
+  echo "Resetting all cloned data."
+  rm -rf "${ANSIBLE_PLUGIN_DIR}"
+  rm -rf "${ANSIBLE_ROLE_DIR}"
+  rm -f  "${ANSIBLE_CFG_PATH}"
+fi
+
+# Download the Ansible plugins repository if it is not present on the host.
+if [ ! -d "${ANSIBLE_PLUGIN_DIR}" ]; then
+  git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              "${ANSIBLE_PLUGIN_DIR}"
+fi
+
+# Download the Ansible role repositories if they are not present on the host.
+# This is ignored if there is no ansible-role-requirements file.
+if [ ! -d "${ANSIBLE_ROLE_DIR}s" ] && [ -f "${ANSIBLE_ROLE_REQUIREMENTS_PATH}" ]; then
+  ansible-galaxy install \
+                 --role-file="${ANSIBLE_ROLE_REQUIREMENTS_PATH}" \
+                 --roles-path "${ANSIBLE_ROLE_DIR}" \
+                 --force
+fi
+
+
+# If a role name is provided, replace the role in the roles folder with a link
+# to the current folder. This ensures that the test executes with the checked
+# out git repo.
+if [ ! -z "${ROLE_NAME}" ]; then
+  echo "Linking ${ANSIBLE_ROLE_DIR}/${ROLE_NAME} to ${WORKING_DIR}"
+  mkdir -p "${ANSIBLE_ROLE_DIR}"
+  rm -rf "${ANSIBLE_ROLE_DIR}/${ROLE_NAME}"
+  ln -s "${WORKING_DIR}" "${ANSIBLE_ROLE_DIR}/${ROLE_NAME}"
+else
+  echo "Skipping the role link because no role name was provided."
+fi
+
+# Ensure that the Ansible configuration file is in the right place
+if [ ! -f "${ANSIBLE_CFG_PATH}" ]; then
+  if [ -f "${COMMON_TESTS_PATH}/test-ansible.cfg" ]; then
+    echo "Linking ${ANSIBLE_CFG_PATH} to ${COMMON_TESTS_PATH}/test-ansible.cfg"
+    ln -s "${COMMON_TESTS_PATH}/test-ansible.cfg" "${ANSIBLE_CFG_PATH}"
+  else
+    echo "Skipping the ansible.cfg link because ${COMMON_TESTS_PATH}/test-ansible.cfg is not there!"
+  fi
+else
+  echo "Found ${ANSIBLE_CFG_PATH} so there's nothing more to do."
+fi
+

test-ansible.cfg

@@ -0,0 +1,29 @@
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This is a set of configuration options for Ansible.
+
+[defaults]
+transport         = ssh
+host_key_checking = False
+control_path      = /tmp/%%h-%%r
+ssh_args          = -o ControlMaster=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=64 -o ServerAliveCountMax=1024 -o Compression=no -o TCPKeepAlive=yes -o VerifyHostKeyDNS=no -o ForwardX11=no -o ForwardAgent=yes
+library           = $HOME/.ansible/plugins/library
+roles_path        = $HOME/.ansible/roles

test-bashate.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script executes bashate against all the files it find that match
+# the search pattern. The search pattern is meant to find any shell
+# scripts present in the role.
+#
+# The test ignores the following rules:
+#
+# E003: Indent not multiple of 4 (we prefer to use multiples of 2)
+#
+# E006: Line longer than 79 columns (as many scripts use jinja
+#       templating, this is very difficult)
+#
+# E040: Syntax error determined using `bash -n` (as many scripts
+#       use jinja templating, this will often fail and the syntax
+#       error will be discovered in execution anyway)
+
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+
+grep --recursive --binary-files=without-match \
+     --files-with-match '^.!.*\(ba\)\?sh$' \
+     --exclude-dir .tox \
+     --exclude-dir .git \
+     "${WORKING_DIR}" | xargs bashate --error . --verbose --ignore=E003,E006,E040

test-log-collect.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script collects, renames and compresses the logs produced in
+# a role test if the host is in OpenStack-CI.
+
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+
+if [[ -d "/etc/nodepool" ]];then
+  mkdir -p "${WORKING_DIR}/logs/host" "${WORKING_DIR}/logs/openstack"
+  rsync --archive --verbose --safe-links --ignore-errors /var/log/ "${WORKING_DIR}/logs/host" || true
+  rsync --archive --verbose --safe-links --ignore-errors /openstack/log/ "${WORKING_DIR}/logs/openstack" || true
+  # Rename all files gathered to have a .txt suffix so that the compressed
+  # files are viewable via a web browser in OpenStack-CI.
+  find "${WORKING_DIR}/logs/" -type f -exec mv {} {}.txt \;
+  # Compress the files gathered so that they do not take up too much space.
+  # We use 'command' to ensure that we're not executing with some sort of alias.
+  command gzip --best --recursive "${WORKING_DIR}/logs/"
+fi

test-pep8.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script executes flake8 against all the files it find that match
+# the search pattern. The search pattern is meant to find any python
+# scripts present in the role.
+
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+
+grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*python$' \
+        --exclude-dir .eggs \
+        --exclude-dir .git \
+        --exclude-dir .tox \
+        --exclude-dir *.egg-info \
+        --exclude-dir doc \
+        "${WORKING_DIR}" | xargs flake8 --verbose

tests/ansible-role-requirements.yml

@@ -10,6 +10,10 @@
   src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
   scm: git
   version: master
+- name: openstack_hosts
+  src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
+  scm: git
+  version: master
 - name: lxc_hosts
   src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
   scm: git
@@ -42,7 +46,3 @@
   src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
   scm: git
   version: master
-- name: openstack_hosts
-  src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
-  scm: git
-  version: master

tests/test-tests-functional.yml

@@ -27,4 +27,4 @@
       environment:
         RUN_TEMPEST_OPTS: "--serial"
   vars_files:
-    - playbooks/test-vars.yml
+    - common/test-vars.yml

tests/test.yml

@@ -14,22 +14,22 @@
 # limitations under the License.
 
 # Prepare the user ssh keys
-- include: playbooks/test-prepare-keys.yml
+- include: common/test-prepare-keys.yml
 
 # Prepare the host
-- include: playbooks/test-prepare-host.yml
+- include: common/test-prepare-host.yml
 
 # Prepare the containers
-- include: playbooks/test-prepare-containers.yml
+- include: common/test-prepare-containers.yml
 
 # Install RabbitMQ/MariaDB/Memcached
-- include: playbooks/test-install-infra.yml
+- include: common/test-install-infra.yml
 
 # Install Keystone
-- include: playbooks/test-install-keystone.yml
+- include: common/test-install-keystone.yml
 
 # Install Tempest
-- include: playbooks/test-install-tempest.yml
+- include: common/test-install-tempest.yml
 
 # Test test repository
 # TODO(andymccr): Add additional tests for now start with just keystone

tox.ini

@@ -22,12 +22,10 @@ passenv =
     NO_PROXY
 whitelist_externals =
     bash
-    git
-    rm
-    wget
 setenv =
-    VIRTUAL_ENV={envdir}
     PYTHONUNBUFFERED=1
+    VIRTUAL_ENV={envdir}
+    WORKING_DIR={toxinidir}
 
 
 [testenv:docs]
@@ -53,17 +51,26 @@ commands =
     {posargs}
 
 
+# The clone URL should be set to the appropriate git URL.
+# In the tests repo itself, the URL is uniquely set to
+# the toxinidir so that the role is able to test itself, but
+# the tox config is exactly the same as other repositories.
+#
+# The URL for other repositories must be:
+# https://git.openstack.org/openstack/openstack-ansible-tests
+# or for a stable branch:
+# -b stable/mitaka https://git.openstack.org/openstack/openstack-ansible-tests
+[testenv:tests_clone]
+commands =
+    bash -c "if [ ! -d "{toxinidir}/tests/common" ]; then \
+               git clone {toxinidir} {toxinidir}/tests/common; \
+             fi"
+
+
 [testenv:pep8]
 commands =
-    # Run hacking/flake8 check for all python files
+    {[testenv:tests_clone]commands}
-    bash -c "grep --recursive --binary-files=without-match \
+    bash -c "{toxinidir}/tests/common/test-pep8.sh"
-        --files-with-match '^.!.*python$' \
-        --exclude-dir .eggs \
-        --exclude-dir .git \
-        --exclude-dir .tox \
-        --exclude-dir *.egg-info \
-        --exclude-dir doc \
-        {toxinidir} | xargs flake8 --verbose"
 
 
 [flake8]
@@ -76,64 +83,31 @@ ignore=F403,H303
 
 [testenv:bashate]
 commands =
-    # Run bashate check for all bash scripts
+    {[testenv:tests_clone]commands}
-    # Ignores the following rules:
+    bash -c "{toxinidir}/tests/common/test-bashate.sh"
-    #     E003: Indent not multiple of 4 (we prefer to use multiples of 2)
-    #     E006: Line longer than 79 columns (as many scripts use jinja
-    #           templating, this is very difficult)
-    #     E040: Syntax error determined using `bash -n` (as many scripts
-    #           use jinja templating, this will often fail and the syntax
-    #           error will be discovered in execution anyway)
-    bash -c "grep --recursive --binary-files=without-match \
-        --files-with-match '^.!.*\(ba\)\?sh$' \
-        --exclude-dir .tox \
-        --exclude-dir .git \
-        {toxinidir} | xargs bashate --error . --verbose --ignore=E003,E006,E040"
 
 
+# The deps URL should be set to the appropriate git URL.
+# In the tests repo itself, the variable is uniquely set to
+# the toxinidir so that the role is able to test itself, but
+# the tox config is exactly the same as other repositories.
+#
+# The value for other repositories must be:
+# http://git.openstack.org/cgit/openstack/openstack-ansible-tests/plain/test-ansible-deps.txt
+# or for a stable branch:
+# http://git.openstack.org/cgit/openstack/openstack-ansible-tests/plain/test-ansible-deps.txt?h=stable/newton
 [testenv:ansible]
 deps =
     {[testenv]deps}
-    ansible==2.1.1
+    -r{toxinidir}/test-ansible-deps.txt
-    ansible-lint>=2.7.0,<3.0.0
-setenv =
-    {[testenv]setenv}
-    ANSIBLE_HOST_KEY_CHECKING = False
-    ANSIBLE_SSH_CONTROL_PATH = /tmp/%%h-%%r
-    # TODO (odyssey4me) These are only here as they are non-standard folder
-    # names for Ansible 1.9.x. We are using the standard folder names for
-    # Ansible v2.x. We can remove this when we move to Ansible 2.x.
-    ANSIBLE_ACTION_PLUGINS = {homedir}/.ansible/plugins/action
-    ANSIBLE_CALLBACK_PLUGINS = {homedir}/.ansible/plugins/callback
-    ANSIBLE_FILTER_PLUGINS = {homedir}/.ansible/plugins/filter
-    ANSIBLE_LOOKUP_PLUGINS = {homedir}/.ansible/plugins/lookup
-    # This is required as the default is the current path or a path specified
-    # in ansible.cfg
-    ANSIBLE_LIBRARY = {homedir}/.ansible/plugins/library
-    # This is required as the default is '/etc/ansible/roles' or a path
-    # specified in ansible.cfg
-    ANSIBLE_ROLES_PATH = {homedir}/.ansible/roles:{toxinidir}/..
-    ANSIBLE_TRANSPORT = "ssh"
-
 commands =
-    rm -rf {homedir}/.ansible/plugins
+    {[testenv:tests_clone]commands}
-    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+    bash -c "{toxinidir}/tests/common/test-ansible-env-prep.sh"
-              {homedir}/.ansible/plugins
-    rm -rf {homedir}/.ansible/roles
-    ansible-galaxy install \
-                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
-                   --force
-    rm -rf {homedir}/.ansible/roles/tests
-    bash -c "ln -s {toxinidir} {homedir}/.ansible/roles/tests"
-    rm -rf {toxinidir}/tests/playbooks
-    git clone . {toxinidir}/tests/playbooks
 
 
 [testenv:ansible-syntax]
 deps =
     {[testenv:ansible]deps}
-setenv =
-    {[testenv:ansible]setenv}
 commands =
     {[testenv:ansible]commands}
     ansible-playbook -i {toxinidir}/tests/inventory \
@@ -160,10 +134,7 @@ install_command =
 
 [testenv:func_logs]
 commands =
-    bash -c 'mkdir -p {toxinidir}/logs'
+    bash -c "{toxinidir}/tests/common/test-log-collect.sh"
-    bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
-    bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
-    bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
 
 
 [testenv:functional]
@@ -179,8 +150,6 @@ install_command =
     {[testenv:func_base]install_command}
 deps =
     {[testenv:ansible]deps}
-setenv =
-    {[testenv:ansible]setenv}
 commands =
     {[testenv:ansible]commands}
     ansible-playbook -i {toxinidir}/tests/inventory \
@@ -191,8 +160,6 @@ commands =
 [testenv:linters]
 deps =
     {[testenv:ansible]deps}
-setenv =
-    {[testenv:ansible]setenv}
 commands =
     {[testenv:pep8]commands}
     {[testenv:bashate]commands}