diff --git a/playbooks/post.yaml b/playbooks/post.yaml index 36023a625a..41d3c54121 100644 --- a/playbooks/post.yaml +++ b/playbooks/post.yaml @@ -21,6 +21,8 @@ - export-devstack-journal - apache-logs-conf - devstack-project-conf + # capture-system-logs should be the last role before stage-output + - capture-system-logs - role: stage-output zuul_copy_output: { '{{ devstack_conf_dir }}/local.conf': 'logs', @@ -40,7 +42,22 @@ '/var/log/mysql.log': 'logs', '/var/log/libvirt': 'logs', '/etc/sudoers': 'logs', - '/etc/sudoers.d': 'logs' } + '/etc/sudoers.d': 'logs', + '{{ stage_dir }}/iptables.txt': 'logs', + '{{ stage_dir }}/df.txt': 'logs', + '{{ stage_dir }}/pip2-freeze.txt': 'logs', + '{{ stage_dir }}/pip3-freeze.txt': 'logs', + '{{ stage_dir }}/dpkg-l.txt': 'logs', + '{{ stage_dir }}/rpm-qa.txt': 'logs', + '{{ stage_dir }}/core': 'logs', + '{{ stage_dir }}/listen53.txt': 'logs', + '{{ stage_dir }}/deprecations.log': 'logs', + '/var/log/ceph': 'logs', + '/var/log/openvswitch': 'logs', + '/var/log/glusterfs': 'logs', + '/etc/glusterfs/glusterd.vol': 'logs', + '/etc/resolv.conf': 'logs', + '/var/log/unbound.log': 'logs' } extensions_to_txt: - conf - log diff --git a/roles/capture-system-logs/README.rst b/roles/capture-system-logs/README.rst new file mode 100644 index 0000000000..c28412457a --- /dev/null +++ b/roles/capture-system-logs/README.rst @@ -0,0 +1,20 @@ +Stage a number of system type logs + +Stage a number of different logs / reports: +- snapshot of iptables +- disk space available +- pip[2|3] freeze +- installed packages (dpkg/rpm) +- ceph, openswitch, gluster +- coredumps +- dns resolver +- listen53 +- unbound.log +- deprecation messages + +**Role Variables** + +.. zuul:rolevar:: stage_dir + :default: {{ ansible_user_dir }} + + The base stage directory. diff --git a/roles/capture-system-logs/defaults/main.yaml b/roles/capture-system-logs/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/capture-system-logs/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/capture-system-logs/tasks/main.yaml b/roles/capture-system-logs/tasks/main.yaml new file mode 100644 index 0000000000..cd8f4f0eab --- /dev/null +++ b/roles/capture-system-logs/tasks/main.yaml @@ -0,0 +1,38 @@ +# TODO(andreaf) Make this into proper Ansible +- name: Stage various logs and reports + shell: + cmd: | + sudo iptables-save > {{ stage_dir }}/iptables.txt + df -h > {{ stage_dir }}/df.txt + + for py_ver in 2 3; do + if [[ `which python${py_ver}` ]]; then + python${py_ver} -m pip freeze > {{ stage_dir }}/pip${py_ver}-freeze.txt + fi + done + + if [ `command -v dpkg` ]; then + dpkg -l> {{ stage_dir }}/dpkg-l.txt + fi + if [ `command -v rpm` ]; then + rpm -qa | sort > {{ stage_dir }}/rpm-qa.txt + fi + + # gzip and save any coredumps in /var/core + if [ -d /var/core ]; then + sudo gzip -r /var/core + sudo cp -r /var/core {{ stage_dir }}/ + fi + + sudo ss -lntup | grep ':53' > {{ stage_dir }}/listen53.txt + + # NOTE(andreaf) Service logs are already in logs/ thanks for the + # export-devstack-journal log. Apache logs are under apache/ thans to the + # apache-logs-conf role. + grep -i deprecat {{ stage_dir }}/logs/*.txt {{ stage_dir }}/apache/*.log | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\.[0-9]{1,3}/ /g' | \ + sed -r 's/[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}/ /g' | \ + sed -r 's/[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}/ /g' | + sed -r 's/\[.*\]/ /g' | \ + sed -r 's/\s[0-9]+\s/ /g' | \ + awk '{if ($0 in seen) {seen[$0]++} else {out[++n]=$0;seen[$0]=1}} END { for (i=1; i<=n; i++) print seen[out[i]]" :: " out[i] }' > {{ stage_dir }}/deprecations.log