Merge "Have run_tests clone the test repo"

.gitignore

@@ -69,3 +69,4 @@ tests/*.retry
 
 # Git clones
 openstack-ansible-ops
+previous

run_tests.sh

@@ -12,8 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Note:
+
+# PURPOSE:
+# This script clones the openstack-ansible-tests repository to the
+# tests/common folder in order to be able to re-use test components
+# for role testing. This is intended to be the thinnest possible
+# shim for test execution outside of OpenStack CI.
+
+# WARNING:
 # This file is maintained in the openstack-ansible-tests repository.
 # https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/run_tests.sh
 # If you need to modify this file, update the one in the openstack-ansible-tests
@@ -26,33 +32,60 @@ set -xeu
 
 ## Vars ----------------------------------------------------------------------
 
-export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+WORKING_DIR="$(readlink -f $(dirname $0))"
+
+COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
+TESTING_HOME=${TESTING_HOME:-$HOME}
+ZUUL_TESTS_CLONE_LOCATION="/home/zuul/src/git.openstack.org/openstack/openstack-ansible-tests"
+
+# Use .gitreview as the key to determine the appropriate
+# branch to clone for tests.
+TESTING_BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' "${WORKING_DIR}/.gitreview")
+if [[ "${TESTING_BRANCH}" == "" ]]; then
+  TESTING_BRANCH="master"
+fi
 
 ## Main ----------------------------------------------------------------------
 
+# Source distribution information
 source /etc/os-release || source /usr/lib/os-release
 
-install_pkg_deps() {
-    pkg_deps="git"
+# Prefer dnf over yum for CentOS.
+which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
 
-    # Prefer dnf over yum for CentOS.
-    which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
-
-    case ${ID,,} in
+# Figure out the appropriate package install command
+case ${ID,,} in
     *suse*) pkg_mgr_cmd="zypper -n in" ;;
     centos|rhel|fedora) pkg_mgr_cmd="${RHT_PKG_MGR} install -y" ;;
     ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;;
     *) echo "unsupported distribution: ${ID,,}"; exit 1 ;;
-    esac
+esac
 
-    eval sudo $pkg_mgr_cmd $pkg_deps
-}
-
-# Install the host distro package dependencies
-install_pkg_deps
+# Install git so that we can clone the tests repo
+eval sudo $pkg_mgr_cmd git
 
 # Clone the tests repo for access to the common test script
-source tests/tests-repo-clone.sh
+if [[ ! -d ${COMMON_TESTS_PATH} ]]; then
+    # The tests repo doesn't need a clone, we can just
+    # symlink it.
+    if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
+        ln -s ${WORKING_DIR} ${COMMON_TESTS_PATH}
+
+    # In zuul v3 any dependent repository is placed into
+    # /home/zuul/src/git.openstack.org, so we check to see
+    # if there is a tests checkout there already. If so, we
+    # symlink that and use it.
+    elif [[ -d "${ZUUL_TESTS_CLONE_LOCATION}" ]]; then
+        ln -s "${ZUUL_TESTS_CLONE_LOCATION}" ${COMMON_TESTS_PATH}
+
+    # Otherwise we're clearly not in zuul or using a previously setup
+    # repo in some way, so just clone it from upstream.
+    else
+        git clone -b ${TESTING_BRANCH} \
+            https://git.openstack.org/openstack/openstack-ansible-tests \
+            ${COMMON_TESTS_PATH}
+    fi
+fi
 
 # Execute the common test script
 source tests/common/run_tests_common.sh

run_tests_common.sh

@@ -13,16 +13,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -o pipefail
-set -euov
+## Shell Opts ----------------------------------------------------------------
 
+set -o pipefail
+set -xeuo
+
+## Prerequisite check --------------------------------------------------------
+
+# Check whether the require environment variables are set.
+if [[ -z ${WORKING_DIR+x} ]] ||\
+   [[ -z ${COMMON_TESTS_PATH+x} ]] ||\
+   [[ -z ${TESTING_HOME+x} ]] ||\
+   [[ -z ${TESTING_BRANCH+x} ]]; then
+     echo "Required environment variables are not set."
+     echo "Please ensure that run_tests.sh is used to execute tests."
+     exit 1
+fi
+
+## Vars ----------------------------------------------------------------------
+
+# Set the source branch for upgrade tests
+# Be sure to change this whenever a new stable branch
+# is created. The checkout must always be N-1.
+UPGRADE_SOURCE_BRANCH=${UPGRADE_SOURCE_BRANCH:-'stable/queens'}
+
+# The bindep file contains the basic distribution packages
+# required in order to install pip, and ansible via pip.
 BINDEP_FILE=${BINDEP_FILE:-bindep.txt}
 
+## Main ----------------------------------------------------------------------
+
+# If this test set includes an upgrade test, the
+# previous stable release tests repo must also be
+# cloned.
+# Note:
+# Dependent patches to the previous stable release
+# tests repo are not supported.
+if [[ ! -d "${COMMON_TESTS_PATH}/previous" ]]; then
+  git clone -b ${UPGRADE_SOURCE_BRANCH} \
+      https://git.openstack.org/openstack/openstack-ansible-tests \
+      ${COMMON_TESTS_PATH}/previous
+fi
+
+# Source distribution information
 source /etc/os-release || source /usr/lib/os-release
 
 # Prefer dnf over yum for CentOS.
 which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
 
+# Perform the initial distribution package install
+# to allow pip and bindep to work.
 case "${ID,,}" in
     *suse*)
         # Need to pull libffi and python-pyOpenSSL early

tox.ini

@@ -56,7 +56,6 @@ commands =
 
 [testenv:pep8]
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-pep8.sh"
 
 
@@ -69,7 +68,6 @@ ignore=F403
 
 [testenv:bashate]
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-bashate.sh"
 
 
@@ -93,7 +91,6 @@ deps =
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-syntax.sh"
 
 
@@ -101,29 +98,13 @@ commands =
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-lint.sh"
 
 
-# This is here due to a bug in the ubuntu-xenial
-# job which does not do the translation from
-# 'func' to 'functional' for us.
-# TODO(odyssey4me):
-# Remove this once https://review.openstack.org/512351
-# has been merged.
-[testenv:func]
-deps =
-    {[testenv:ansible]deps}
-commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
-    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
-
-
 [testenv:functional]
 deps =
     {[testenv:ansible]deps}
 commands =
-    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
     bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
 
 

zuul.d/jobs.yaml

@@ -13,56 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- job:
-    name: openstack-ansible-linters
-    parent: openstack-tox-linters
-    description: |
-      Run lint tests for an OpenStack-Ansible project.
-      Uses tox with the ``linters`` environment.
-    required-projects:
-      - name: openstack/ansible-hardening
-      - name: openstack/ansible-role-python_venv_build
-      - name: openstack/ansible-role-systemd_mount
-      - name: openstack/ansible-role-systemd_networkd
-      - name: openstack/ansible-role-systemd_service
-      - name: openstack/openstack-ansible-apt_package_pinning
-      - name: openstack/openstack-ansible-ceph_client
-      - name: openstack/openstack-ansible-galera_client
-      - name: openstack/openstack-ansible-galera_server
-      - name: openstack/openstack-ansible-haproxy_server
-      - name: openstack/openstack-ansible-lxc_container_create
-      - name: openstack/openstack-ansible-lxc_hosts
-      - name: openstack/openstack-ansible-memcached_server
-      - name: openstack/openstack-ansible-openstack_hosts
-      - name: openstack/openstack-ansible-openstack_openrc
-      - name: openstack/openstack-ansible-ops
-      - name: openstack/openstack-ansible-os_aodh
-      - name: openstack/openstack-ansible-os_barbican
-      - name: openstack/openstack-ansible-os_ceilometer
-      - name: openstack/openstack-ansible-os_cinder
-      - name: openstack/openstack-ansible-os_designate
-      - name: openstack/openstack-ansible-os_glance
-      - name: openstack/openstack-ansible-os_gnocchi
-      - name: openstack/openstack-ansible-os_heat
-      - name: openstack/openstack-ansible-os_keystone
-      - name: openstack/openstack-ansible-os_magnum
-      - name: openstack/openstack-ansible-os_neutron
-      - name: openstack/openstack-ansible-os_nova
-      - name: openstack/openstack-ansible-os_octavia
-      - name: openstack/openstack-ansible-os_swift
-      - name: openstack/openstack-ansible-os_tempest
-      - name: openstack/openstack-ansible-os_trove
-      - name: openstack/openstack-ansible-pip_install
-      - name: openstack/openstack-ansible-plugins
-      - name: openstack/openstack-ansible-rabbitmq_server
-      - name: openstack/openstack-ansible-repo_build
-      - name: openstack/openstack-ansible-repo_server
-      - name: openstack/openstack-ansible-rsyslog_client
-      - name: openstack/openstack-ansible-rsyslog_server
-      - name: openstack/openstack-ansible-tests
-      - name: openstack/requirements
-    timeout: 600 # 5 mins
-
 - job:
     name: openstack-ansible-functional
     parent: base
@@ -127,6 +77,18 @@
     vars:
       tox_env: functional
 
+- job:
+    name: openstack-ansible-linters
+    parent: openstack-ansible-functional
+    description: |
+      Run lint tests for an OpenStack-Ansible project.
+      Uses tox with the ``linters`` environment.
+    irrelevant-files: []
+    timeout: 600 # 5 mins
+    nodeset: ubuntu-xenial
+    vars:
+      tox_env: linters
+
 - job:
     name: openstack-ansible-upgrade
     parent: openstack-ansible-functional