Update tox config and add bashate E006, E040 exceptions

This patch updates the tox.ini the same bashate exceptions as are
currently in the OpenStack-Ansible playbook repo.

It also ensures that the linters and all lint targets work
appropriately and normalises the tox.ini configuration to use
uniform formatting.

The use of ansible.cfg is removed as there is no way of being
certain which paths can be used without reverting to an ugly
sed hack in the commands. This is why it is preferred to make
use of environment variables which make use of tox's default
substitutions instead. It's a more reliable way of achieving
the goal for the purpose of gating and testing.

The switch to using a git clone instead of ansible-galaxy to
download the plugins is due to the path spec not being able to
work in Ansible 2.x. [1]

[1] https://github.com/ansible/ansible/issues/13563

Change-Id: I1df29fe1f9c2ee2e5370c4dd9f733ae2c697608e

test-requirements.txt

@@ -1,5 +1,7 @@
 ansible-lint
 ansible>=1.9.1,<2.0.0
+bashate
+flake8
 
 # this is required for the docs build jobs
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2

tests/ansible.cfg

@@ -1,3 +0,0 @@
-[defaults]
-roles_path = ../../
-remote_tmp = ../.ansible/tmp/

tests/test.yml

@@ -38,11 +38,6 @@
           name: "trusty.tgz"
           sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
           chroot_path: trusty/rootfs-amd64
-      # The $HOME directory is mocked to work with tox
-      #  by defining the 'ansible_env' hash. This should
-      #  NEVER be done outside of testing.
-      ansible_env:  ## NEVER DO THIS OUTSIDE OF TESTING
-        HOME: "/tmp"
   post_tasks:
     - name: Open sysctl file
       slurp:

tox.ini

@@ -3,109 +3,123 @@ minversion = 1.6
 skipsdist = True
 envlist = docs,linters,functional
 
+
 [testenv]
 usedevelop = True
 install_command = pip install -U {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
 commands = /usr/bin/find . -type f -name "*.pyc" -delete
+passenv =
+    HOME
+whitelist_externals =
+    bash
+    git
+    rm
+setenv =
+    VIRTUAL_ENV={envdir}
+    ANSIBLE_HOST_KEY_CHECKING = False
+    ANSIBLE_SSH_CONTROL_PATH = /tmp/%%h-%%r
+    # TODO (odyssey4me) These are only here as they are non-standard folder
+    # names for Ansible 1.9.x. We are using the standard folder names for
+    # Ansible v2.x. We can remove this when we move to Ansible 2.x.
+    ANSIBLE_ACTION_PLUGINS = {homedir}/.ansible/plugins/action
+    ANSIBLE_CALLBACK_PLUGINS = {homedir}/.ansible/plugins/callback
+    ANSIBLE_FILTER_PLUGINS = {homedir}/.ansible/plugins/filter
+    ANSIBLE_LOOKUP_PLUGINS = {homedir}/.ansible/plugins/lookup
+    # This is required as the default is the current path or a path specified
+    # in ansible.cfg
+    ANSIBLE_LIBRARY = {homedir}/.ansible/plugins/library
+    # This is required as the default is '/etc/ansible/roles' or a path
+    # specified in ansible.cfg
+    ANSIBLE_ROLES_PATH = {homedir}/.ansible/roles:{toxinidir}/..
+
 
 [testenv:docs]
 commands=
     python setup.py build_sphinx
 
+
 # environment used by the -infra templated docs job
 [testenv:venv]
 deps = -r{toxinidir}/test-requirements.txt
 commands = {posargs}
 
+
 [testenv:pep8]
-deps =
-    flake8
-whitelist_externals = bash
 commands =
     # Run hacking/flake8 check for all python files
-    bash -c "grep -Irl \
-        -e '!/usr/bin/env python' \
-        -e '!/bin/python' \
-        -e '!/usr/bin/python' \
-        --exclude-dir '.*' \
-        --exclude-dir 'doc' \
-        --exclude-dir '*.egg' \
-        --exclude-dir '*.egg-info' \
-        --exclude 'tox.ini' \
-        --exclude '*.sh' \
+    bash -c "grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*python$' \
+        --exclude-dir .eggs \
+        --exclude-dir .git \
+        --exclude-dir .tox \
+        --exclude-dir *.egg-info \
+        --exclude-dir doc \
         {toxinidir} | xargs flake8 --verbose"
 
 [flake8]
 # Ignores the following rules due to how ansible modules work in general
 #     F403 'from ansible.module_utils.basic import *' used;
-#           unable to detect undefined names
-#     H303  No wildcard (*) import.
+#          unable to detect undefined names
+#     H303 No wildcard (*) import.
 ignore=F403,H303
 
+
 [testenv:bashate]
-deps =
-  bashate
-whitelist_externals = bash
 commands =
-  # Run bashate check for all bash scripts
-  # Ignores the following rules:
-  #     E003: Indent not multiple of 4 (we prefer to use multiples of 2)
-  bash -c "grep --recursive --binary-files=without-match \
-    --files-with-match '^.!.*\(ba\)\?sh$' \
-    --exclude-dir .tox \
-    --exclude-dir .git \
-    {toxinidir} | xargs bashate --error . --verbose --ignore=E003"
+    # Run bashate check for all bash scripts
+    # Ignores the following rules:
+    #     E003: Indent not multiple of 4 (we prefer to use multiples of 2)
+    #     E006: Line longer than 79 columns (as many scripts use jinja
+    #           templating, this is very difficult)
+    #     E040: Syntax error determined using `bash -n` (as many scripts
+    #           use jinja templating, this will often fail and the syntax
+    #           error will be discovered in execution anyway)
+    bash -c "grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*\(ba\)\?sh$' \
+        --exclude-dir .tox \
+        --exclude-dir .git \
+        {toxinidir} | xargs bashate --error . --verbose --ignore=E003,E006,E040"
+
 
 [testenv:ansible-syntax]
-deps =
-  ansible>1.9,<2.0
-changedir = tests
 commands =
+    rm -rf {homedir}/.ansible
+    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              {homedir}/.ansible/plugins
     ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
+                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
                    --ignore-errors \
                    --force
-    ansible-playbook -i inventory \
+    ansible-playbook -i {toxinidir}/tests/inventory \
                      --syntax-check \
                      --list-tasks \
                      -e "rolename={toxinidir}" \
-                     test.yml
+                     {toxinidir}/tests/test.yml
+
 
 [testenv:ansible-lint]
-deps =
-  ansible>1.9,<2.0
-  ansible-lint
-changedir = tests
 commands =
-    ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
-                   --ignore-errors \
-                   --force
-    ansible-lint test.yml
+    ansible-lint {toxinidir}/tests/test.yml
+
 
 [testenv:functional]
-changedir = tests
 commands =
+    rm -rf {homedir}/.ansible
+    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              {homedir}/.ansible/plugins
     ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
+                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
                    --ignore-errors \
                    --force
-    ansible-playbook -i inventory \
+    ansible-playbook -i {toxinidir}/tests/inventory \
                      -e "rolename={toxinidir}" \
-                     test.yml
+                     {toxinidir}/tests/test.yml
+
 
 [testenv:linters]
-deps =
-  {[testenv:pep8]deps}
-  {[testenv:bashate]deps}
-  {[testenv:ansible-lint]deps}
-  # note that the ansible-syntax deps are omitted on purpose as the
-  #  ansible-lint env contains duplicate items in its dep list
-whitelist_externals = bash
 commands =
-  {[testenv:pep8]commands}
-  {[testenv:bashate]commands}
-  {[testenv:ansible-lint]commands}
-  {[testenv:ansible-syntax]commands}
+    {[testenv:pep8]commands}
+    {[testenv:bashate]commands}
+    {[testenv:ansible-lint]commands}
+    {[testenv:ansible-syntax]commands}