Stage a number of system type logs

This could be ansiblesed a more, for now more or less the bash code
from d-g wrapped in a role.

Change-Id: Ia7fe40f05ca394da9a65fef6383d72f29a532e2f

playbooks/post.yaml

@@ -21,6 +21,8 @@
     - export-devstack-journal
     - apache-logs-conf
     - devstack-project-conf
+    # capture-system-logs should be the last role before stage-output
+    - capture-system-logs
     - role: stage-output
       zuul_copy_output:
         { '{{ devstack_conf_dir }}/local.conf': 'logs',
@@ -40,7 +42,22 @@
           '/var/log/mysql.log': 'logs',
           '/var/log/libvirt': 'logs',
           '/etc/sudoers': 'logs',
-          '/etc/sudoers.d': 'logs' }
+          '/etc/sudoers.d': 'logs',
+          '{{ stage_dir }}/iptables.txt': 'logs',
+          '{{ stage_dir }}/df.txt': 'logs',
+          '{{ stage_dir }}/pip2-freeze.txt': 'logs',
+          '{{ stage_dir }}/pip3-freeze.txt': 'logs',
+          '{{ stage_dir }}/dpkg-l.txt': 'logs',
+          '{{ stage_dir }}/rpm-qa.txt': 'logs',
+          '{{ stage_dir }}/core': 'logs',
+          '{{ stage_dir }}/listen53.txt': 'logs',
+          '{{ stage_dir }}/deprecations.log': 'logs',
+          '/var/log/ceph': 'logs',
+          '/var/log/openvswitch': 'logs',
+          '/var/log/glusterfs': 'logs',
+          '/etc/glusterfs/glusterd.vol': 'logs',
+          '/etc/resolv.conf': 'logs',
+          '/var/log/unbound.log': 'logs' }
       extensions_to_txt:
         - conf
         - log

roles/capture-system-logs/README.rst

@@ -0,0 +1,20 @@
+Stage a number of system type logs
+
+Stage a number of different logs / reports:
+- snapshot of iptables
+- disk space available
+- pip[2|3] freeze
+- installed packages (dpkg/rpm)
+- ceph, openswitch, gluster
+- coredumps
+- dns resolver
+- listen53
+- unbound.log
+- deprecation messages
+
+**Role Variables**
+
+.. zuul:rolevar:: stage_dir
+   :default: {{ ansible_user_dir }}
+
+   The base stage directory.

roles/capture-system-logs/defaults/main.yaml

@@ -0,0 +1 @@
+devstack_base_dir: /opt/stack

roles/capture-system-logs/tasks/main.yaml

@@ -0,0 +1,38 @@
+# TODO(andreaf) Make this into proper Ansible
+- name: Stage various logs and reports
+  shell:
+    cmd: |
+      sudo iptables-save > {{ stage_dir }}/iptables.txt
+      df -h > {{ stage_dir }}/df.txt
+
+      for py_ver in 2 3; do
+          if [[ `which python${py_ver}` ]]; then
+              python${py_ver} -m pip freeze > {{ stage_dir }}/pip${py_ver}-freeze.txt
+          fi
+      done
+
+      if [ `command -v dpkg` ]; then
+          dpkg -l> {{ stage_dir }}/dpkg-l.txt
+      fi
+      if [ `command -v rpm` ]; then
+          rpm -qa | sort > {{ stage_dir }}/rpm-qa.txt
+      fi
+
+      # gzip and save any coredumps in /var/core
+      if [ -d /var/core ]; then
+          sudo gzip -r /var/core
+          sudo cp -r /var/core {{ stage_dir }}/
+      fi
+
+      sudo ss -lntup | grep ':53' > {{ stage_dir }}/listen53.txt
+
+      # NOTE(andreaf) Service logs are already in logs/ thanks for the
+      # export-devstack-journal log. Apache logs are under apache/ thans to the
+      # apache-logs-conf role.
+      grep -i deprecat {{ stage_dir }}/logs/*.txt {{ stage_dir }}/apache/*.log | \
+          sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \
+          sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \
+          sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' |
+          sed -r 's/\[.*\]/ /g' | \
+          sed -r 's/\s[0-9]+\s/ /g' | \
+          awk '{if ($0 in seen) {seen[$0]++} else {out[++n]=$0;seen[$0]=1}} END { for (i=1; i<=n; i++) print seen[out[i]]" :: " out[i] }' > {{ stage_dir }}/deprecations.log