From 3f6190cb654b2cf8ec1ed9840bd64a202cbdffb0 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 1 Aug 2018 16:56:12 -0700 Subject: [PATCH] Add HTMLify logs role A simple role to make HTML versions of all files with .txt extensions. Co-Authored-By: Tobias Henkel Change-Id: Ia5feac28a23df37965e50de726a1caa821b546ad --- roles/htmlify-logs/README.rst | 5 + roles/htmlify-logs/library/__init__.py | 0 roles/htmlify-logs/library/htmlify.py | 139 ++++++++++++++++++ .../library/test-fixtures/in/job-output.txt | 54 +++++++ .../test-fixtures/in/job-output.txt.gz | Bin 0 -> 1267 bytes .../test-fixtures/reference/job-output.txt | 112 ++++++++++++++ roles/htmlify-logs/library/test_htmlify.py | 48 ++++++ roles/htmlify-logs/tasks/main.yaml | 14 ++ 8 files changed, 372 insertions(+) create mode 100644 roles/htmlify-logs/README.rst create mode 100644 roles/htmlify-logs/library/__init__.py create mode 100644 roles/htmlify-logs/library/htmlify.py create mode 100644 roles/htmlify-logs/library/test-fixtures/in/job-output.txt create mode 100644 roles/htmlify-logs/library/test-fixtures/in/job-output.txt.gz create mode 100644 roles/htmlify-logs/library/test-fixtures/reference/job-output.txt create mode 100644 roles/htmlify-logs/library/test_htmlify.py create mode 100644 roles/htmlify-logs/tasks/main.yaml diff --git a/roles/htmlify-logs/README.rst b/roles/htmlify-logs/README.rst new file mode 100644 index 000000000..336bdedee --- /dev/null +++ b/roles/htmlify-logs/README.rst @@ -0,0 +1,5 @@ +HTMLify text logs + +This makes an HTML version of every file in the executor log directory +with a ``.txt`` or ``.txt.gz`` extension. If the original was +gzipped, the HTML version will be as well. diff --git a/roles/htmlify-logs/library/__init__.py b/roles/htmlify-logs/library/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/htmlify-logs/library/htmlify.py b/roles/htmlify-logs/library/htmlify.py new file mode 100644 index 000000000..a97b9b94d --- /dev/null +++ b/roles/htmlify-logs/library/htmlify.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# Copyright 2018 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import cgi +import argparse +import gzip +import sys + +from ansible.module_utils.basic import AnsibleModule + +HEADER = ''' + + + + +
+'''
+
+FOOTER = '''
+ + + +''' + + +def run(inpath, outpath): + if inpath.endswith('.gz'): + infile = gzip.open(inpath, 'r') + outfile = gzip.open(outpath, 'w') + else: + infile = open(inpath, 'r') + outfile = open(outpath, 'w') + + outfile.write(HEADER) + + for i, line in enumerate(infile): + i = i + 1 + line = cgi.escape(line.rstrip('\n')) + outfile.write('' % (i, i, i)) + outfile.write(line.rstrip('\n')) + outfile.write("\n") + + outfile.write(FOOTER) + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + input=dict(required=True, type='path'), + output=dict(required=True, type='path'), + ) + ) + + p = module.params + run(p.get('input'), p.get('output')) + + module.exit_json(changed=True) + + +def cli_main(): + parser = argparse.ArgumentParser( + description="HTMLify text files" + ) + parser.add_argument('input', + help='Input path') + parser.add_argument('output', + help='Output path') + + args = parser.parse_args() + + run(args.input, args.output) + + +if __name__ == '__main__': + + if sys.stdin.isatty(): + cli_main() + else: + ansible_main() diff --git a/roles/htmlify-logs/library/test-fixtures/in/job-output.txt b/roles/htmlify-logs/library/test-fixtures/in/job-output.txt new file mode 100644 index 000000000..f0a585123 --- /dev/null +++ b/roles/htmlify-logs/library/test-fixtures/in/job-output.txt @@ -0,0 +1,54 @@ +2018-08-01 00:43:51.328884 | Job console starting... +2018-08-01 00:44:01.742989 | PRE-RUN START: [trusted : git.openstack.org/openstack-infra/project-config/playbooks/base/pre.yaml@master] +2018-08-01 00:44:04.693529 | +2018-08-01 00:44:04.693765 | PLAY [localhost] +2018-08-01 00:44:04.736664 | +2018-08-01 00:44:04.736863 | TASK [emit-job-header : Setup log path fact] +2018-08-01 00:44:04.804077 | localhost | ok +2018-08-01 00:44:04.870027 | +2018-08-01 00:44:04.870264 | TASK [set-zuul-log-path-fact : Set log path for a change] +2018-08-01 00:44:04.942854 | localhost | skipping: Conditional result was False +2018-08-01 00:44:04.970178 | +2018-08-01 00:44:04.970405 | TASK [set-zuul-log-path-fact : Set log path for a ref update] +2018-08-01 00:44:05.067095 | localhost | ok +2018-08-01 00:44:05.094186 | +2018-08-01 00:44:05.094430 | TASK [set-zuul-log-path-fact : Set log path for a periodic job] +2018-08-01 00:44:05.152841 | localhost | skipping: Conditional result was False +2018-08-01 00:44:05.188754 | +2018-08-01 00:44:05.189030 | TASK [emit-job-header : Print job information] +2018-08-01 00:44:05.317656 | # Job Information +2018-08-01 00:44:05.318162 | Ansible Version: 2.5.7 +2018-08-01 00:44:05.318269 | Job: trigger-readthedocs-webhook +2018-08-01 00:44:05.318371 | Pipeline: post +2018-08-01 00:44:05.318469 | Executor: ze09.openstack.org +2018-08-01 00:44:05.318567 | Triggered by: https://git.openstack.org/cgit/openstack/gerrit-dash-creator/commit/?id=ee5167a518de07c325f326df212c92e5c1e786a9 +2018-08-01 00:44:05.318663 | Log URL (when completed): http://logs.openstack.org/ee/ee5167a518de07c325f326df212c92e5c1e786a9/post/trigger-readthedocs-webhook/f677f03 +2018-08-01 00:44:05.349043 | +2018-08-01 00:44:05.349250 | PLAY [all] +2018-08-01 00:44:05.385083 | +2018-08-01 00:44:05.385308 | TASK [Gather network facts] +2018-08-01 00:44:07.104842 | ubuntu-xenial | ok +2018-08-01 00:44:07.167964 | +2018-08-01 00:44:07.168193 | TASK [add-build-sshkey : Check to see if ssh key was already created for this build] +2018-08-01 00:44:07.760495 | ubuntu-xenial -> localhost | ok +2018-08-01 00:44:07.799900 | +2018-08-01 00:44:07.800163 | TASK [add-build-sshkey : Create Temp SSH key] +2018-08-01 00:44:08.445211 | ubuntu-xenial -> localhost | Generating public/private rsa key pair. +2018-08-01 00:44:08.445859 | ubuntu-xenial -> localhost | Your identification has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa. +2018-08-01 00:44:08.445989 | ubuntu-xenial -> localhost | Your public key has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.pub. +2018-08-01 00:44:08.446099 | ubuntu-xenial -> localhost | The key fingerprint is: +2018-08-01 00:44:08.446205 | ubuntu-xenial -> localhost | SHA256:ZNugwiDo1mjkhNQt/GnNoH3XQgo+uU/jA8NtHUKCPiY zuul@ze09 +2018-08-01 00:44:08.446320 | ubuntu-xenial -> localhost | The key's randomart image is: +2018-08-01 00:44:08.446424 | ubuntu-xenial -> localhost | +---[RSA 1024]----+ +2018-08-01 00:44:08.446527 | ubuntu-xenial -> localhost | | .o o | +2018-08-01 00:44:08.446630 | ubuntu-xenial -> localhost | |+ = = . . | +2018-08-01 00:44:08.446732 | ubuntu-xenial -> localhost | |+oo * X * . | +2018-08-01 00:44:08.446834 | ubuntu-xenial -> localhost | |=EoB O X B . | +2018-08-01 00:44:08.446936 | ubuntu-xenial -> localhost | | *o.* * S + | +2018-08-01 00:44:08.447037 | ubuntu-xenial -> localhost | |o * = . | +2018-08-01 00:44:08.447138 | ubuntu-xenial -> localhost | | B . | +2018-08-01 00:44:08.447239 | ubuntu-xenial -> localhost | | + | +2018-08-01 00:44:08.447339 | ubuntu-xenial -> localhost | | . | +2018-08-01 00:44:08.447440 | ubuntu-xenial -> localhost | +----[SHA256]-----+ +2018-08-01 00:44:08.447643 | ubuntu-xenial -> localhost | ok: Runtime: 0:00:00.022230 +here is a line with some diff --git a/roles/htmlify-logs/library/test-fixtures/in/job-output.txt.gz b/roles/htmlify-logs/library/test-fixtures/in/job-output.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..cc76bd0e7840868e83e50479cff24ea2042fa43c GIT binary patch literal 1267 zcmVaUva=bY)c@CA}NUy)~;@sHCu};jgz5W zP!zO8Ta0B<5T&Lm^5b`u+w>(?6chty*?{dMpL?F;-SH?1g-7rm0wEb^Srqz=3L!XL z!=GjWxv8zG6xdF-ovv5D??0elZ{isV{e;t0q`2w$5i_)&&gReITy1pcU6_Se zDp{x&_0Cn-H&B@sG_osUDf1COf^d=~h;+jTznk^$wFx0K8RLXjIxyaHva0hw^nK+a z3D1%69EmNpFNm2IB;=*6S86xGlv5G$ZwT78ZW_#p44#_0(4985te{o4uR6Gt7JiqN zRlD&L5+-6y1X?+XzDq=_me4na>~;!?d=e)ljqVd^T$}Px#JfojR?f(Gk!)0}O`&tZ zY}rjSjHuw@e-&@U6C#O564_Z(@`aS!iG1AZx^rv+%ha?R>3r{I&O)p&7qlOS?(xUx z^&`tg7*kw6tF2yOoB3O{78hqg{m4(o_Ru)pwxX?;tlDJelBET|87O3gbjZp(qqdaz>*iqj9mMAas*!XQ5fS4kB4R|`B>pojQOx&Rb(f}H5B=Kif=&F8 zaKT;L_KUvmdhfleHP+%tE5Y?~l8#*@&MLz6^F<8#hFn04#V#0?u_%(m;(R5$L&1sqvGA2e462^Cn z8HmEE+B7hqA37nsSPGx>h=$=5%V(-qt#qRaH2tE|InD|Co%7IIIYgP|0pJ~T5ZG4czAWn<+#47#o%cqiMS(u=@d1Hqu#e#-EWiZJ9%NKq`< zQZkW7EaqvR1TNDiSH0E6TNE>@VLNzDYuLtTh{^xLAN`MMh)FuB;j~l(_a&aSYMbFw z(l#4Iq+~L~=Z7;I#o0ec{pwQxX2Q+Ix;%Q>1<&dubI4x3T$%m;EV!77qwetR`O{w`UaDsNd!FiEn#Jzn&DP`mV&J3<_aj + + + + +
+2018-08-01 00:43:51.328884 | Job console starting...
+2018-08-01 00:44:01.742989 | PRE-RUN START: [trusted : git.openstack.org/openstack-infra/project-config/playbooks/base/pre.yaml@master]
+2018-08-01 00:44:04.693529 | 
+2018-08-01 00:44:04.693765 | PLAY [localhost]
+2018-08-01 00:44:04.736664 | 
+2018-08-01 00:44:04.736863 | TASK [emit-job-header : Setup log path fact]
+2018-08-01 00:44:04.804077 | localhost | ok
+2018-08-01 00:44:04.870027 | 
+2018-08-01 00:44:04.870264 | TASK [set-zuul-log-path-fact : Set log path for a change]
+2018-08-01 00:44:04.942854 | localhost | skipping: Conditional result was False
+2018-08-01 00:44:04.970178 | 
+2018-08-01 00:44:04.970405 | TASK [set-zuul-log-path-fact : Set log path for a ref update]
+2018-08-01 00:44:05.067095 | localhost | ok
+2018-08-01 00:44:05.094186 | 
+2018-08-01 00:44:05.094430 | TASK [set-zuul-log-path-fact : Set log path for a periodic job]
+2018-08-01 00:44:05.152841 | localhost | skipping: Conditional result was False
+2018-08-01 00:44:05.188754 | 
+2018-08-01 00:44:05.189030 | TASK [emit-job-header : Print job information]
+2018-08-01 00:44:05.317656 | # Job Information
+2018-08-01 00:44:05.318162 | Ansible Version: 2.5.7
+2018-08-01 00:44:05.318269 | Job: trigger-readthedocs-webhook
+2018-08-01 00:44:05.318371 | Pipeline: post
+2018-08-01 00:44:05.318469 | Executor: ze09.openstack.org
+2018-08-01 00:44:05.318567 | Triggered by: https://git.openstack.org/cgit/openstack/gerrit-dash-creator/commit/?id=ee5167a518de07c325f326df212c92e5c1e786a9
+2018-08-01 00:44:05.318663 | Log URL (when completed): http://logs.openstack.org/ee/ee5167a518de07c325f326df212c92e5c1e786a9/post/trigger-readthedocs-webhook/f677f03
+2018-08-01 00:44:05.349043 | 
+2018-08-01 00:44:05.349250 | PLAY [all]
+2018-08-01 00:44:05.385083 | 
+2018-08-01 00:44:05.385308 | TASK [Gather network facts]
+2018-08-01 00:44:07.104842 | ubuntu-xenial | ok
+2018-08-01 00:44:07.167964 | 
+2018-08-01 00:44:07.168193 | TASK [add-build-sshkey : Check to see if ssh key was already created for this build]
+2018-08-01 00:44:07.760495 | ubuntu-xenial -> localhost | ok
+2018-08-01 00:44:07.799900 | 
+2018-08-01 00:44:07.800163 | TASK [add-build-sshkey : Create Temp SSH key]
+2018-08-01 00:44:08.445211 | ubuntu-xenial -> localhost | Generating public/private rsa key pair.
+2018-08-01 00:44:08.445859 | ubuntu-xenial -> localhost | Your identification has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.
+2018-08-01 00:44:08.445989 | ubuntu-xenial -> localhost | Your public key has been saved in /var/lib/zuul/builds/f677f0312811438a8db3fa38953649c7/work/f677f0312811438a8db3fa38953649c7_id_rsa.pub.
+2018-08-01 00:44:08.446099 | ubuntu-xenial -> localhost | The key fingerprint is:
+2018-08-01 00:44:08.446205 | ubuntu-xenial -> localhost | SHA256:ZNugwiDo1mjkhNQt/GnNoH3XQgo+uU/jA8NtHUKCPiY zuul@ze09
+2018-08-01 00:44:08.446320 | ubuntu-xenial -> localhost | The key's randomart image is:
+2018-08-01 00:44:08.446424 | ubuntu-xenial -> localhost | +---[RSA 1024]----+
+2018-08-01 00:44:08.446527 | ubuntu-xenial -> localhost | | .o o            |
+2018-08-01 00:44:08.446630 | ubuntu-xenial -> localhost | |+  = = . .       |
+2018-08-01 00:44:08.446732 | ubuntu-xenial -> localhost | |+oo * X * .      |
+2018-08-01 00:44:08.446834 | ubuntu-xenial -> localhost | |=EoB O X B .     |
+2018-08-01 00:44:08.446936 | ubuntu-xenial -> localhost | | *o.* * S +      |
+2018-08-01 00:44:08.447037 | ubuntu-xenial -> localhost | |o    * = .       |
+2018-08-01 00:44:08.447138 | ubuntu-xenial -> localhost | |      B .        |
+2018-08-01 00:44:08.447239 | ubuntu-xenial -> localhost | |       +         |
+2018-08-01 00:44:08.447339 | ubuntu-xenial -> localhost | |        .        |
+2018-08-01 00:44:08.447440 | ubuntu-xenial -> localhost | +----[SHA256]-----+
+2018-08-01 00:44:08.447643 | ubuntu-xenial -> localhost | ok: Runtime: 0:00:00.022230
+here is a line with some <escapes>
+
+ + + diff --git a/roles/htmlify-logs/library/test_htmlify.py b/roles/htmlify-logs/library/test_htmlify.py new file mode 100644 index 000000000..a9abebc60 --- /dev/null +++ b/roles/htmlify-logs/library/test_htmlify.py @@ -0,0 +1,48 @@ +# Copyright (C) 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import os + +import testtools +import fixtures + +from .htmlify import run + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'test-fixtures') + + +class TestHTMLify(testtools.TestCase): + + def _test_file(self, fn): + in_path = os.path.join(FIXTURE_DIR, 'in', fn) + ref_path = os.path.join(FIXTURE_DIR, 'reference', fn) + out_root = self.useFixture(fixtures.TempDir()).path + out_path = os.path.join(out_root, fn) + run(in_path, out_path) + + if fn.endswith('gz'): + out = gzip.open(out_path, 'rb') + else: + out = open(out_path, 'rb') + + generated_data = out.read() + reference_data = open(ref_path, 'rb').read() + self.assertEqual(reference_data, generated_data) + + def test_htmlify(self): + self._test_file('job-output.txt') diff --git a/roles/htmlify-logs/tasks/main.yaml b/roles/htmlify-logs/tasks/main.yaml new file mode 100644 index 000000000..50bd7c6d1 --- /dev/null +++ b/roles/htmlify-logs/tasks/main.yaml @@ -0,0 +1,14 @@ +- name: Find text files to HTMLify + find: + paths: "{{ zuul.executor.log_root }}" + recurse: true + patterns: + - "*.txt" + - "*.txt.gz" + register: htmlify_files + +- name: HTMLify text files + htmlify: + input: "{{ item.path }}" + output: "{{ item.path | regex_replace('\\.txt', '.html') }}" + loop: "{{ htmlify_files.files }}"