Fix idempotency check and add debug logging

The current idempotency check is broken because Ansible
does not produce a log output without the environment
variable being exported.

This patch ensures that each playbook run produces an
individual log, and ensures that the idempotency check
uses the correct log output.

The following are also included:
- The script outputs the full CLI used to execute the test
  playbook. This is useful for debugging purposes.
- The parameters are re-ordered in alphabetical order to
  make them easier to find.
- The log collection now also collects the Ansible logs.
- Instead of repeating the rsync command in the log collection
  a variable is used, reducing the line length and making it
  easier to change/re-use.

Change-Id: Ie379de765c6ebba958ce8e7f9dc27b7a3af74ff8

test-ansible-env-prep.sh

@@ -34,10 +34,11 @@ export WORKING_DIR=${WORKING_DIR:-$(pwd)}
 export ROLE_NAME=${ROLE_NAME:-''}
 export ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY:-$WORKING_DIR/tests/inventory}
 
-export ANSIBLE_NOCOLOR=1
-export ANSIBLE_ROLE_DIR="${TESTING_HOME}/.ansible/roles"
-export ANSIBLE_PLUGIN_DIR="${TESTING_HOME}/.ansible/plugins"
 export ANSIBLE_CFG_PATH="${TESTING_HOME}/.ansible.cfg"
+export ANSIBLE_LOG_DIR="${TESTING_HOME}/.ansible/logs"
+export ANSIBLE_NOCOLOR=1
+export ANSIBLE_PLUGIN_DIR="${TESTING_HOME}/.ansible/plugins"
+export ANSIBLE_ROLE_DIR="${TESTING_HOME}/.ansible/roles"
 export ANSIBLE_ROLE_REQUIREMENTS_PATH="${WORKING_DIR}/tests/ansible-role-requirements.yml"
 export COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
 
@@ -58,11 +59,15 @@ export PYTHONUNBUFFERED=1
 # If the test reset toggle is set, destroy the existing cloned data.
 if [ "${TEST_RESET}" == "true" ]; then
   echo "Resetting all cloned data."
+  rm -f  "${ANSIBLE_CFG_PATH}"
+  rm -rf "${ANSIBLE_LOG_DIR}"
   rm -rf "${ANSIBLE_PLUGIN_DIR}"
   rm -rf "${ANSIBLE_ROLE_DIR}"
-  rm -f  "${ANSIBLE_CFG_PATH}"
 fi
 
+# Create the directory which will hold all Ansible logs
+mkdir -p "${ANSIBLE_LOG_DIR}"
+
 # Download the Ansible plugins repository if it is not present on the host.
 if [ ! -d "${ANSIBLE_PLUGIN_DIR}" ]; then
   git clone https://git.openstack.org/openstack/openstack-ansible-plugins \

test-ansible-functional.sh

@@ -50,12 +50,21 @@ echo "TEST_IDEMPOTENCE: ${TEST_IDEMPOTENCE}"
 ## Functions -----------------------------------------------------------------
 
 function set_ansible_parameters {
+
   if [ -f "${ANSIBLE_OVERRIDES}" ]; then
     ANSIBLE_CLI_PARAMETERS="${ANSIBLE_PARAMETERS} -e @${ANSIBLE_OVERRIDES}"
   else
     ANSIBLE_CLI_PARAMETERS="${ANSIBLE_PARAMETERS}"
   fi
-  echo "ANSIBLE_CLI_PARAMETERS: ${ANSIBLE_CLI_PARAMETERS}"
+
+}
+
+function execute_ansible_playbook {
+
+  CMD_TO_EXECUTE="ansible-playbook ${ANSIBLE_CLI_PARAMETERS} ${TEST_PLAYBOOK} $@"
+  echo "Executing: ${CMD_TO_EXECUTE}"
+  ${CMD_TO_EXECUTE}
+
 }
 
 function gate_job_exit_tasks {
@@ -75,14 +84,20 @@ set_ansible_parameters
 
 # If the test for check mode is enabled, then execute it
 if [ "${TEST_CHECK_MODE}" == "true" ]; then
-  ansible-playbook --check \
-                   ${ANSIBLE_CLI_PARAMETERS} \
-                   ${TEST_PLAYBOOK}
+
+  # Set the path for the output log
+  export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-check.log"
+
+  # Execute the test playbook in check mode
+  execute_ansible_playbook --check
+
 fi
 
+# Set the path for the output log
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute.log"
+
 # Execute the test playbook
-ansible-playbook ${ANSIBLE_CLI_PARAMETERS} \
-                 ${TEST_PLAYBOOK}
+execute_ansible_playbook
 
 # If the idempotence test is enabled, then execute the
 # playbook again and verify that nothing changed/failed
@@ -91,14 +106,13 @@ ansible-playbook ${ANSIBLE_CLI_PARAMETERS} \
 if [ "${TEST_IDEMPOTENCE}" == "true" ]; then
 
   # Set the path for the output log
-  ANSIBLE_LOG_PATH="/tmp/ansible.log"
+  export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-idempotence.log"
 
   # Execute the test playbook
-  ansible-playbook ${ANSIBLE_CLI_PARAMETERS} \
-                   ${TEST_PLAYBOOK}
+  execute_ansible_playbook
 
   # Check the output log for changed/failed tasks
-  if grep -q "changed=0.*failed=0" /tmp/ansible.log; then
+  if grep -q "changed=0.*failed=0" ${ANSIBLE_LOG_PATH}; then
     echo "Idempotence test: pass"
   else
     echo "Idempotence test: fail"

test-log-collect.sh

@@ -26,13 +26,20 @@
 ## Vars ----------------------------------------------------------------------
 
 export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+export RSYNC_CMD="rsync --archive --verbose --safe-links --ignore-errors"
 
 ## Main ----------------------------------------------------------------------
 
 if [[ -d "/etc/nodepool" ]]; then
   mkdir -p "${WORKING_DIR}/logs/host" "${WORKING_DIR}/logs/openstack"
-  rsync --archive --verbose --safe-links --ignore-errors /var/log/ "${WORKING_DIR}/logs/host" || true
-  rsync --archive --verbose --safe-links --ignore-errors /openstack/log/ "${WORKING_DIR}/logs/openstack" || true
+  ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true
+  ${RSYNC_CMD} /openstack/log/ "${WORKING_DIR}/logs/openstack" || true
+
+  if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then
+    mkdir -p "${WORKING_DIR}/logs/ansible"
+    ${RSYNC_CMD} "${ANSIBLE_LOG_DIR}/" "${WORKING_DIR}/logs/ansible" || true
+  fi
+
   # Rename all files gathered to have a .txt suffix so that the compressed
   # files are viewable via a web browser in OpenStack-CI.
   find "${WORKING_DIR}/logs/" -type f -exec mv {} {}.txt \;