Ansible 2.1.1 role testing

This commit also moves the upgrade test to its own
scenario using tox environment func_upgrade

Closes-Bug: #1623592
Change-Id: Ib79d9c261c3bf5e018806867ebe804b5853908e9
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

.gitignore

@@ -62,6 +62,8 @@ releasenotes/build
 
 # Test temp files
 tests/plugins
+tests/playbooks
+tests/test.retry
 
 # Vagrant artifacts
 .vagrant

manual-test.rc

@@ -0,0 +1,33 @@
+export VIRTUAL_ENV=$(pwd)
+export ANSIBLE_HOST_KEY_CHECKING=False
+export ANSIBLE_SSH_CONTROL_PATH=/tmp/%%h-%%r
+
+# TODO (odyssey4me) These are only here as they are non-standard folder
+# names for Ansible 1.9.x. We are using the standard folder names for
+# Ansible v2.x. We can remove this when we move to Ansible 2.x.
+export ANSIBLE_ACTION_PLUGINS=${HOME}/.ansible/plugins/action
+export ANSIBLE_CALLBACK_PLUGINS=${HOME}/.ansible/plugins/callback
+export ANSIBLE_FILTER_PLUGINS=${HOME}/.ansible/plugins/filter
+export ANSIBLE_LOOKUP_PLUGINS=${HOME}/.ansible/plugins/lookup
+
+# This is required as the default is the current path or a path specified
+# in ansible.cfg
+export ANSIBLE_LIBRARY=${HOME}/.ansible/plugins/library
+
+# This is required as the default is '/etc/ansible/roles' or a path
+# specified in ansible.cfg
+export ANSIBLE_ROLES_PATH=${HOME}/.ansible/roles:$(pwd)/..
+
+export ANSIBLE_SSH_ARGS="-o ControlMaster=no \
+  -o UserKnownHostsFile=/dev/null \
+  -o StrictHostKeyChecking=no \
+  -o ServerAliveInterval=64 \
+  -o ServerAliveCountMax=1024 \
+  -o Compression=no \
+  -o TCPKeepAlive=yes \
+  -o VerifyHostKeyDNS=no \
+  -o ForwardX11=no \
+  -o ForwardAgent=yes"
+
+echo "Run manual functional tests by executing the following:"
+echo "# ./.tox/functional/bin/ansible-playbook -i tests/inventory tests/test.yml -e \"rolename=$(pwd)\""

run_tests.sh

@@ -24,23 +24,23 @@ if [ ! "$(which pip)" ]; then
 fi
 
 # Install bindep and tox
-pip install bindep tox
+sudo pip install bindep tox
 
 # CentOS 7 requires two additional packages:
 #   redhat-lsb-core - for bindep profile support
 #   epel-release    - required to install python-ndg_httpsclient/python2-pyasn1
 if [ "$(which yum)" ]; then
-    yum -y install redhat-lsb-core epel-release
+    sudo yum -y install redhat-lsb-core epel-release
 fi
 
 # Install OS packages using bindep
 if apt-get -v >/dev/null 2>&1 ; then
-    apt-get update
+    sudo apt-get update
     DEBIAN_FRONTEND=noninteractive \
-      apt-get -q --option "Dpkg::Options::=--force-confold" \
+      sudo apt-get -q --option "Dpkg::Options::=--force-confold" \
       --assume-yes install `bindep -b -f bindep.txt test`
 else
-    yum install -y `bindep -b -f bindep.txt test`
+    sudo yum install -y `bindep -b -f bindep.txt test`
 fi
 
 # run through each tox env and execute the test

tasks/rabbitmq_pre_install.yml

@@ -44,7 +44,7 @@
   lineinfile:
     dest: /etc/hosts
     state: present
-    line: "{{ hostvars[item]['ansible_ssh_host'] }} {{ hostvars[item]['ansible_hostname'] }} {{ hostvars[item]['ansible_hostname'].split('.')[0] }}"
+    line: "{{ hostvars[item]['ansible_host'] }} {{ hostvars[item]['ansible_hostname'] }} {{ hostvars[item]['ansible_hostname'].split('.')[0] }}"
   with_items: "{{ groups[rabbitmq_host_group] }}"
   tags:
     - rabbitmq-config

tests/ansible-role-requirements.yml

@@ -6,6 +6,10 @@
   src: https://git.openstack.org/openstack/openstack-ansible-pip_install
   scm: git
   version: master
+- name: "openstack_hosts"
+  src: "https://github.com/openstack/openstack-ansible-openstack_hosts"
+  scm: git
+  version: "master"
 - name: lxc_hosts
   src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
   scm: git

tests/host_vars/localhost.yml

@@ -0,0 +1,19 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bridges:
+  - "br-mgmt"
+
+ansible_python_interpreter: "/usr/bin/python2"

tests/inventory

@@ -1,8 +1,8 @@
 [all]
-localhost ansible_connection=local ansible_become=True
-container1 ansible_ssh_host=10.100.100.101 ansible_host=10.100.100.101 ansible_become=True ansible_user=root
-container2 ansible_ssh_host=10.100.100.102 ansible_host=10.100.100.102 ansible_become=True ansible_user=root
-container3 ansible_ssh_host=10.100.100.103 ansible_host=10.100.100.103 ansible_become=True ansible_user=root
+localhost ansible_become=True
+container1 ansible_host=10.100.100.101 ansible_become=True ansible_user=root
+container2 ansible_host=10.100.100.102 ansible_become=True ansible_user=root
+container3 ansible_host=10.100.100.103 ansible_become=True ansible_user=root
 
 [rabbitmq_all_other]
 container1

tests/test-install-rabbitmq-server.yml

@@ -13,14 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Gather facts
+  hosts: "all"
+  user: root
+  gather_facts: true
+
 # NOTE(mancdaz): rabbitmq cannot be upgraded in serial, so when
 # rabbitmq_upgrade=True, serial is set to 0, else it is 1 for installs
 - name: Install RabbitMQ server
   hosts: "{{ rabbitmq_host_group }}"
   serial: "{{ rabbitmq_upgrade|default(false) | bool | ternary(0, 1)}}"
   user: root
-  gather_facts: true
+  gather_facts: True
   roles:
-    - role: "{{ rolename | basename }}"
+    - role: rabbitmq_server
   vars_files:
     - test-vars.yml

tests/test-prepare-containers.yml

@@ -1,52 +0,0 @@
----
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Create test containers
-  hosts: all_containers
-  gather_facts: false
-  pre_tasks:
-    - name: Destroy test containers
-      lxc_container:
-        name: "{{ container_name }}"
-        state: "absent"
-      delegate_to: "{{ physical_host }}"
-      tags:
-        - container-destroy
-    - name: Destroy container service directories
-      file:
-        path: "{{ item }}"
-        state: "absent"
-      with_items:
-        - "/openstack/{{ container_name }}"
-        - "/openstack/backup/{{ container_name }}"
-        - "/openstack/log/{{ container_name }}"
-        - "/var/lib/lxc/{{ container_name }}"
-        - "{{ lxc_container_directory|default('/var/lib/lxc') }}/{{ container_name }}"
-      delegate_to: "{{ physical_host }}"
-      tags:
-        - container-directories
-  roles:
-    - role: "lxc_container_create"
-      lxc_container_backing_store: dir
-      global_environment_variables:
-        PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-  post_tasks:
-    - name: Wait for ssh to be available
-      local_action:
-        module: wait_for
-        port: "{{ ansible_ssh_port | default('22') }}"
-        host: "{{ ansible_ssh_host | default(inventory_hostname) }}"
-        search_regex: OpenSSH
-        delay: 1

tests/test-prepare-host.yml

@@ -1,52 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Perform basic LXC host setup
-  hosts: localhost
-  pre_tasks:
-    - name: Ensure apt cache is always refreshed
-      apt:
-        update_cache: yes
-      when: ansible_pkg_mgr == 'apt'
-    - name: Ensure yum packages are up to date
-      yum:
-        name: '*'
-        state: latest
-        update_cache: yes
-      when: ansible_pkg_mgr == 'yum'
-    - name: Ensure root's new public ssh key is in authorized_keys
-      authorized_key:
-        user: root
-        key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
-        manage_dir: no
-    - set_fact:
-        lxc_container_ssh_key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
-    - name: Check if this is an OpenStack-CI nodepool instance
-      stat:
-        path: /etc/nodepool/provider
-      register: nodepool
-    - name: Set the files to copy into the container cache for OpenStack-CI instances
-      set_fact:
-        lxc_container_cache_files:
-          - { src: '/etc/pip.conf', dest: '/etc/pip.conf' }
-      when:
-        - nodepool.stat.exists | bool
-  roles:
-    - role: "lxc_hosts"
-      lxc_net_address: 10.100.100.1
-      lxc_net_dhcp_range: 10.100.100.2,10.100.100.100
-      lxc_net_bridge: lxcbr0
-      lxc_kernel_options:
-        - { key: 'fs.inotify.max_user_instances', value: 1024 }

tests/test-prepare-keys.yml

@@ -1,33 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# NOTE: we use become_user because setting become: no or become: false
-#       doesn't seem to override the ansible_become=true in the
-#       inventory
-- name: Create ssh key pairs for use with containers
-  hosts: localhost
-  become_user: "{{ ansible_ssh_user }}"
-  tasks:
-    - name: Create ssh key pair for root
-      user:
-        name: "{{ ansible_ssh_user }}"
-        generate_ssh_key: "yes"
-        ssh_key_bits: 2048
-        ssh_key_file: ".ssh/id_rsa"
-    - name: get the calling users key
-      command: cat ~/.ssh/id_rsa.pub
-      register: key_get
-    - set_fact:
-        lxc_container_ssh_key: "{{ key_get.stdout }}"

tests/test-rabbitmq-server-functional.yml

@@ -17,13 +17,14 @@
   hosts: "{{ rabbitmq_host_group }}"
   user: root
   gather_facts: true
+  become: true
   vars_files:
     - test-vars.yml
   tasks:
     - name: Check rabbitmq is running
       command: "pgrep -f rabbit"
       delegate_to: "{{ item }}"
-      when: ansible_ssh_host == item
+      when: ansible_host == item
       with_items:
         - 10.100.100.101
         - 10.100.100.102

tests/test-upgrade.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setup the host
+- include: playbooks/test-install-openstack-hosts.yml
+- include: playbooks/test-setup-host.yml
+
+# Install previous version of RabbitMQ server
+- include: test-install-rabbitmq-server.yml
+  vars:
+    rabbitmq_package_url: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_url'] }}"
+    rabbitmq_package_sha256: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_sha256'] }}"
+
+# Run tests
+- include: test-rabbitmq-server-functional.yml
+  vars:
+    rabbitmq_release_version: "3.6.1"
+
+# Perform upgrade of RabbitMQ server (package_version will come
+# from role defaults)
+- include: test-install-rabbitmq-server.yml
+  vars:
+    rabbitmq_upgrade: true
+
+# Run tests
+- include: test-rabbitmq-server-functional.yml
+  vars:
+    rabbitmq_release_version: "3.6.5"

tests/test.yml

@@ -13,14 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Prepare the user ssh keys
-- include: test-prepare-keys.yml
-
-# Prepare the host
-- include: test-prepare-host.yml
-
-# Run container clean up and build
-- include: test-prepare-containers.yml
+# Setup the host
+- include: playbooks/test-install-openstack-hosts.yml
+- include: playbooks/test-setup-host.yml
 
 # Install RabbitMQ server
 - include: test-install-rabbitmq-server.yml
@@ -30,29 +25,3 @@
   vars:
     rabbitmq_release_version: "3.6.5"
 
-############################# UPGRADE TESTING #############################
-
-# Run container clean up and build
-- include: test-prepare-containers.yml
-
-# Install previous version of RabbitMQ server
-- include: test-install-rabbitmq-server.yml
-  vars:
-    rabbitmq_package_url: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_url'] }}"
-    rabbitmq_package_sha256: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_sha256'] }}"
-
-# Run tests
-- include: test-rabbitmq-server-functional.yml
-  vars:
-    rabbitmq_release_version: "3.6.1"
-
-# Perform upgrade of RabbitMQ server (package_version will come
-# from role defaults)
-- include: test-install-rabbitmq-server.yml
-  vars:
-    rabbitmq_upgrade: true
-
-# Run tests
-- include: test-rabbitmq-server-functional.yml
-  vars:
-    rabbitmq_release_version: "3.6.5"

tox.ini

@@ -94,7 +94,7 @@ commands =
 [testenv:ansible]
 deps =
     {[testenv]deps}
-    ansible==1.9.4
+    ansible==2.1.1
     ansible-lint>=2.7.0,<3.0.0
 setenv =
     {[testenv]setenv}
@@ -113,6 +113,7 @@ setenv =
     # This is required as the default is '/etc/ansible/roles' or a path
     # specified in ansible.cfg
     ANSIBLE_ROLES_PATH = {homedir}/.ansible/roles:{toxinidir}/..
+    ANSIBLE_TRANSPORT = "ssh"
 commands =
     rm -rf {homedir}/.ansible/plugins
     git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
@@ -121,6 +122,11 @@ commands =
     ansible-galaxy install \
                    --role-file={toxinidir}/tests/ansible-role-requirements.yml \
                    --force
+    rm -rf {homedir}/.ansible/roles/rabbitmq_server
+    bash -c "ln -s {toxinidir} {homedir}/.ansible/roles/rabbitmq_server"
+    rm -rf {toxinidir}/tests/playbooks
+    git clone https://git.openstack.org/openstack/openstack-ansible-tests \
+              {toxinidir}/tests/playbooks
 
 
 [testenv:ansible-syntax]
@@ -133,7 +139,6 @@ commands =
     ansible-playbook -i {toxinidir}/tests/inventory \
                      --syntax-check \
                      --list-tasks \
-                     -e "rolename={toxinidir}" \
                      {toxinidir}/tests/test.yml
 
 
@@ -144,6 +149,21 @@ commands =
     {[testenv:ansible]commands}
     ansible-lint {toxinidir}
 
+[testenv:func_base]
+# NOTE(odyssey4me): this target does not use constraints because
+# it doesn't work in OpenStack-CI yet. Once that's fixed, we can
+# drop the install_command.
+install_command =
+    pip install -U --force-reinstall {opts} {packages}
+
+
+[testenv:func_logs]
+commands =
+    bash -c 'mkdir -p {toxinidir}/logs'
+    bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
+    bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
+
 
 [testenv:functional]
 # Ignore_errors is set to true so that the logs are collected at the
@@ -155,7 +175,7 @@ ignore_errors = True
 # it doesn't work in OpenStack-CI yet. Once that's fixed, we can
 # drop the install_command.
 install_command =
-    pip install -U --force-reinstall {opts} {packages}
+    {[testenv:func_base]install_command}
 deps =
     {[testenv:ansible]deps}
 setenv =
@@ -163,14 +183,31 @@ setenv =
 commands =
     {[testenv:ansible]commands}
     ansible-playbook -i {toxinidir}/tests/inventory \
-                     -e "rolename={toxinidir}" \
                      -e "install_test_packages=True" \
                      {toxinidir}/tests/test.yml -vvvv
-    bash -c 'mkdir -p {toxinidir}/logs'
-    bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
-    bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
-    bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
+    {[testenv:func_logs]commands}
 
+[testenv:func_upgrade]
+# Ignore_errors is set to true so that the logs are collected at the
+#  end of the run. This will not produce a false positive. Any
+#  exception will be mark the run as failed and exit 1 after all of
+#  the commands have been iterated through.
+ignore_errors = True
+# NOTE(odyssey4me): this target does not use constraints because
+# it doesn't work in OpenStack-CI yet. Once that's fixed, we can
+# drop the install_command.
+install_command =
+    {[testenv:func_base]install_command}
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv:ansible]setenv}
+commands =
+    {[testenv:ansible]commands}
+    ansible-playbook -i {toxinidir}/tests/inventory \
+                     -e "install_test_packages=True" \
+                     {toxinidir}/tests/test-upgrade.yml -vvvv
+    {[testenv:func_logs]commands}
 
 [testenv:linters]
 deps =