From a74ff558167653f0010b1a7e002ad56f6e84db1c Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 19 Jul 2019 18:10:54 -0700 Subject: [PATCH] Add generate-zuul-manifest role Also, pin gitpython since it's gone py3-only. Change-Id: I0d626945908ec4df785aea793f6243c6fdfbdb14 --- doc/source/log-roles.rst | 1 + roles/generate-zuul-manifest/README.rst | 28 ++++ roles/generate-zuul-manifest/__init__.py | 0 .../generate-zuul-manifest/defaults/main.yaml | 4 + .../library/__init__.py | 0 .../library/generate_manifest.py | 124 +++++++++++++++++ .../test-fixtures/artifacts/foo.tar.gz | Bin 0 -> 115 bytes .../library/test-fixtures/artifacts/foo.tgz | Bin 0 -> 115 bytes .../links/controller/service_log.txt | 0 .../test-fixtures/links/job-output.json | 1 + .../links/symlink_loop/placeholder | 0 .../logs/controller/compressed.gz | Bin 0 -> 31 bytes .../logs/controller/cpu-load.svg | 3 + .../test-fixtures/logs/controller/journal.xz | Bin 0 -> 32 bytes .../logs/controller/service_log.txt | 0 .../logs/controller/subdir/subdir.txt | 0 .../test-fixtures/logs/controller/syslog | 0 .../test-fixtures/logs/job-output.json | 1 + .../logs/zuul-info/inventory.yaml | 0 .../logs/zuul-info/zuul-info.controller.txt | 0 .../library/test_generate_manifest.py | 128 ++++++++++++++++++ roles/generate-zuul-manifest/tasks/main.yaml | 14 ++ test-playbooks/generate-zuul-manifest.yaml | 60 ++++++++ test-requirements.txt | 1 + zuul-tests.d/general-roles-jobs.yaml | 9 ++ 25 files changed, 374 insertions(+) create mode 100644 roles/generate-zuul-manifest/README.rst create mode 100644 roles/generate-zuul-manifest/__init__.py create mode 100644 roles/generate-zuul-manifest/defaults/main.yaml create mode 100644 roles/generate-zuul-manifest/library/__init__.py create mode 100644 roles/generate-zuul-manifest/library/generate_manifest.py create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tar.gz create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tgz create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/links/controller/service_log.txt create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/links/job-output.json create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/links/symlink_loop/placeholder create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/compressed.gz create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/cpu-load.svg create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/journal.xz create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/service_log.txt create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/subdir/subdir.txt create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/controller/syslog create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/job-output.json create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/inventory.yaml create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/zuul-info.controller.txt create mode 100644 roles/generate-zuul-manifest/library/test_generate_manifest.py create mode 100644 roles/generate-zuul-manifest/tasks/main.yaml create mode 100644 test-playbooks/generate-zuul-manifest.yaml diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst index 395f2426b..579a421de 100644 --- a/doc/source/log-roles.rst +++ b/doc/source/log-roles.rst @@ -5,6 +5,7 @@ Log Roles .. zuul:autorole:: ara-report .. zuul:autorole:: ensure-output-dirs .. zuul:autorole:: fetch-output +.. zuul:autorole:: generate-zuul-manifest .. zuul:autorole:: htmlify-logs .. zuul:autorole:: merge-output-to-logs .. zuul:autorole:: publish-artifacts-to-fileserver diff --git a/roles/generate-zuul-manifest/README.rst b/roles/generate-zuul-manifest/README.rst new file mode 100644 index 000000000..16dd9266a --- /dev/null +++ b/roles/generate-zuul-manifest/README.rst @@ -0,0 +1,28 @@ +Generate a Zuul manifest file for log uploading + +This generates a manifest file in preparation for uploading along +with logs. The Zuul web interface can fetch this file in order to +display logs from a build. + + +**Role Variables** + +.. zuul:rolevar:: generate_zuul_manifest_root + :default: {{ zuul.executor.log_dir }} + + The root directory to index. + +.. zuul:rolevar:: generate_zuul_manifest_filename + :default: zuul-manifest.json + + The name of the manifest file. + +.. zuul:rolevar:: generate_zuul_manifest_output + :default: {{ zuul.executor.log_dir }}/{{ generate_zuul_manifest_filename }} + + The path to the output manifest file. + +.. zuul:rolevar:: generate_zuul_manifest_type + :default: zuul_manifest + + The artifact type to return to Zuul. diff --git a/roles/generate-zuul-manifest/__init__.py b/roles/generate-zuul-manifest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/defaults/main.yaml b/roles/generate-zuul-manifest/defaults/main.yaml new file mode 100644 index 000000000..6a214a05e --- /dev/null +++ b/roles/generate-zuul-manifest/defaults/main.yaml @@ -0,0 +1,4 @@ +generate_zuul_manifest_root: "{{ zuul.executor.log_dir }}" +generate_zuul_manifest_filename: "zuul-manifest.json" +generate_zuul_manifest_output: "{{ zuul.executor.log_dir }}/{{ generate_zuul_manifest_filename }}" +generate_zuul_manifest_type: "zuul_manifest" diff --git a/roles/generate-zuul-manifest/library/__init__.py b/roles/generate-zuul-manifest/library/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/generate_manifest.py b/roles/generate-zuul-manifest/library/generate_manifest.py new file mode 100644 index 000000000..378607f45 --- /dev/null +++ b/roles/generate-zuul-manifest/library/generate_manifest.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import json +import logging +import mimetypes +import os +import stat +import sys + +from ansible.module_utils.basic import AnsibleModule + + +mimetypes.init() + + +def path_in_tree(root, path): + full_path = os.path.realpath(os.path.abspath( + os.path.expanduser(path))) + if not full_path.startswith(root): + logging.debug("Skipping path outside root: %s" % (path,)) + return False + return True + + +def walk(root, original_root=None): + if original_root is None: + original_root = root + logging.debug("Walk: %s", root) + data = [] + dirs = [] + files = [] + for e in os.listdir(root): + if os.path.isdir(os.path.join(root, e)): + if not os.path.islink(os.path.join(root, e)): + dirs.append(e) + else: + files.append(e) + for d in sorted(dirs): + logging.debug("Directory: %s", d) + path = os.path.join(root, d) + if not path_in_tree(original_root, path): + continue + data.append(dict(name=d, + mimetype='application/directory', + encoding=None, + children=walk(os.path.join(root, d), original_root))) + for f in sorted(files): + logging.debug("File: %s", f) + path = os.path.join(root, f) + if not path_in_tree(original_root, path): + continue + mime_guess, encoding = mimetypes.guess_type(path) + if not mime_guess: + mime_guess = 'text/plain' + st = os.stat(path) + last_modified = st[stat.ST_MTIME] + size = st[stat.ST_SIZE] + data.append(dict(name=f, + mimetype=mime_guess, + encoding=encoding, + last_modified=last_modified, + size=size)) + return data + + +def run(root_path, output): + data = walk(root_path, root_path) + with open(output, 'w') as f: + f.write(json.dumps({'tree': data})) + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + root=dict(type='path'), + output=dict(type='path'), + ) + ) + + p = module.params + run(p.get('root'), p.get('output')) + + module.exit_json(changed=True) + + +def cli_main(): + parser = argparse.ArgumentParser( + description="Generate a Zuul file manifest" + ) + parser.add_argument('--verbose', action='store_true', + help='show debug information') + parser.add_argument('root', + help='Root of upload directory') + parser.add_argument('output', + help='Output file path') + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + run(args.root, args.output) + + +if __name__ == '__main__': + if sys.stdin.isatty(): + cli_main() + else: + ansible_main() diff --git a/roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tar.gz b/roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9b1579d90d8178e9a1196bc1b4d87b032cc2eeb2 GIT binary patch literal 115 zcmb2|=3wAb4T@%9etXW4i^-6I<$#T<{)CuG#_t6WniWhJp1;1SsW)QoRiDjA^EOVu z$N$*u(7a2BH(pEk%k`W8Q*2pK(OS#t*PLb*ol-rr<@M_6xw6Yuj}^Q-T^harece|E R82I2`e(ByZSq2RT1_1P;G1344 literal 0 HcmV?d00001 diff --git a/roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tgz b/roles/generate-zuul-manifest/library/test-fixtures/artifacts/foo.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ca9fccb9934364912861d8ed720ca0efb9cab910 GIT binary patch literal 115 zcmb2|=3rn~4T@%9etXW4i^-6I<$#T<{)CuG#_t6WniWhJp1;1SsW)QoRiDjA^EOVu z$N$*u(7a2BH(pEk%k`W8Q*2pK(OS#t*PLb*ol-rr<@M_6xw6Yuj}^Q-T^harece|E R82I2`e(ByZSq2RT1_1L_G0y-1 literal 0 HcmV?d00001 diff --git a/roles/generate-zuul-manifest/library/test-fixtures/links/controller/service_log.txt b/roles/generate-zuul-manifest/library/test-fixtures/links/controller/service_log.txt new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/links/job-output.json b/roles/generate-zuul-manifest/library/test-fixtures/links/job-output.json new file mode 100644 index 000000000..c8cd7e92d --- /dev/null +++ b/roles/generate-zuul-manifest/library/test-fixtures/links/job-output.json @@ -0,0 +1 @@ +{"test": "foo"} diff --git a/roles/generate-zuul-manifest/library/test-fixtures/links/symlink_loop/placeholder b/roles/generate-zuul-manifest/library/test-fixtures/links/symlink_loop/placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/compressed.gz b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/compressed.gz new file mode 100644 index 0000000000000000000000000000000000000000..4dc3bad6630cb3a93498446472cd7636cc572230 GIT binary patch literal 31 fcmb2|=HRFk3yNl7PR`FQC`v6ZPEBE8W`F?zd)@{& literal 0 HcmV?d00001 diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/cpu-load.svg b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/cpu-load.svg new file mode 100644 index 000000000..01a940a25 --- /dev/null +++ b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/cpu-load.svg @@ -0,0 +1,3 @@ + + + diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/journal.xz b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/journal.xz new file mode 100644 index 0000000000000000000000000000000000000000..ea28d9e05f69d458ccfaf4aa985d30d366bfbc25 GIT binary patch literal 32 kcmexsUKJ6=z`*kC+7>qkAdtE5qA0)Zb1fr?!x9+<0JCTbcmMzZ literal 0 HcmV?d00001 diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/service_log.txt b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/service_log.txt new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/subdir/subdir.txt b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/subdir/subdir.txt new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/syslog b/roles/generate-zuul-manifest/library/test-fixtures/logs/controller/syslog new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/job-output.json b/roles/generate-zuul-manifest/library/test-fixtures/logs/job-output.json new file mode 100644 index 000000000..c8cd7e92d --- /dev/null +++ b/roles/generate-zuul-manifest/library/test-fixtures/logs/job-output.json @@ -0,0 +1 @@ +{"test": "foo"} diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/inventory.yaml b/roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/inventory.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/zuul-info.controller.txt b/roles/generate-zuul-manifest/library/test-fixtures/logs/zuul-info/zuul-info.controller.txt new file mode 100644 index 000000000..e69de29bb diff --git a/roles/generate-zuul-manifest/library/test_generate_manifest.py b/roles/generate-zuul-manifest/library/test_generate_manifest.py new file mode 100644 index 000000000..99acdde32 --- /dev/null +++ b/roles/generate-zuul-manifest/library/test_generate_manifest.py @@ -0,0 +1,128 @@ +# Copyright (C) 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import testtools +import fixtures + +from .generate_manifest import walk + + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'test-fixtures') + + +class SymlinkFixture(fixtures.Fixture): + links = [ + ('bad_symlink', '/etc'), + ('bad_symlink_file', '/etc/issue'), + ('good_symlink', 'controller'), + ('recursive_symlink', '.'), + ('symlink_file', 'job-output.json'), + ('symlink_loop_a', 'symlink_loop'), + ('symlink_loop/symlink_loop_b', '..'), + ] + + def _setUp(self): + self._cleanup() + for (src, target) in self.links: + path = os.path.join(FIXTURE_DIR, 'links', src) + os.symlink(target, path) + self.addCleanup(self._cleanup) + + def _cleanup(self): + for (src, target) in self.links: + path = os.path.join(FIXTURE_DIR, 'links', src) + if os.path.exists(path): + os.unlink(path) + + +class TestFileList(testtools.TestCase): + + def flatten(self, result, out=None, path=''): + if out is None: + out = [] + dirs = [] + for x in result: + x['_relative_path'] = os.path.join(path, x['name']) + out.append(x) + if 'children' in x: + dirs.append(x) + for x in dirs: + self.flatten(x['children'], out, x['_relative_path']) + x.pop('children') + return out + + def assert_files(self, root, result, files): + self.assertEqual(len(result), len(files)) + for expected, received in zip(files, result): + self.assertEqual(expected[0], received['_relative_path']) + if expected[0] and expected[0][-1] == '/': + efilename = os.path.split( + os.path.dirname(expected[0]))[1] + '/' + else: + efilename = os.path.split(expected[0])[1] + self.assertEqual(efilename, received['name']) + full_path = os.path.join(root, received['_relative_path']) + if received['mimetype'] == 'application/directory': + self.assertTrue(os.path.isdir(full_path)) + else: + self.assertTrue(os.path.isfile(full_path)) + self.assertEqual(expected[1], received['mimetype']) + self.assertEqual(expected[2], received['encoding']) + + def find_file(self, file_list, path): + for f in file_list: + if f.relative_path == path: + return f + + def test_single_dir(self): + '''Test a single directory with a trailing slash''' + + root = os.path.join(FIXTURE_DIR, 'logs') + fl = walk(root) + self.assert_files(root, self.flatten(fl), [ + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ]) + + def test_symlinks(self): + '''Test symlinks''' + self.useFixture(SymlinkFixture()) + root = os.path.join(FIXTURE_DIR, 'links') + fl = walk(root) + self.assert_files(root, self.flatten(fl), [ + ('controller', 'application/directory', None), + ('symlink_loop', 'application/directory', None), + ('job-output.json', 'application/json', None), + ('symlink_file', 'text/plain', None), + ('controller/service_log.txt', 'text/plain', None), + ('symlink_loop/placeholder', 'text/plain', None), + ]) diff --git a/roles/generate-zuul-manifest/tasks/main.yaml b/roles/generate-zuul-manifest/tasks/main.yaml new file mode 100644 index 000000000..aa440f495 --- /dev/null +++ b/roles/generate-zuul-manifest/tasks/main.yaml @@ -0,0 +1,14 @@ +- name: Generate Zuul manifest + generate_manifest: + root: "{{ generate_zuul_manifest_root }}" + output: "{{ generate_zuul_manifest_output }}" + +- name: Return Zuul manifest URL to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: Manifest + url: "{{ generate_zuul_manifest_filename }}" + metadata: + type: "{{ generate_zuul_manifest_type }}" diff --git a/test-playbooks/generate-zuul-manifest.yaml b/test-playbooks/generate-zuul-manifest.yaml new file mode 100644 index 000000000..3b571bf18 --- /dev/null +++ b/test-playbooks/generate-zuul-manifest.yaml @@ -0,0 +1,60 @@ +- name: Run tests for the generate-zuul-manifest role + hosts: all + pre_tasks: + - name: Create test directories + file: + path: "{{ ansible_user_dir }}/{{ item }}" + state: directory + loop: + - tests + - tests/logs + + - name: Create tests files + copy: + dest: "{{ ansible_user_dir }}/{{ item }}" + content: "" + loop: + - tests/index.txt + - tests/logs/file.txt + - tests/logs/file.png + + roles: + - role: generate-zuul-manifest + generate_zuul_manifest_root: "{{ ansible_user_dir }}/tests" + generate_zuul_manifest_filename: "test-manifest.json" + generate_zuul_manifest_output: "{{ ansible_user_dir }}/tests/{{ generate_zuul_manifest_filename }}" + generate_zuul_manifest_type: "test_zuul_manifest" + + post_tasks: + - name: Fetch output + fetch: + src: "{{ ansible_user_dir }}/tests/test-manifest.json" + flat: true + dest: "{{ zuul.executor.log_root }}/" + + - name: Load output + include_vars: + file: "{{ zuul.executor.log_root }}/test-manifest.json" + name: manifest + + - name: Check output + vars: + got: "{{ manifest['tree'] }}" + exp: + - name: logs + mimetype: application/directory + children: + - name: file.png + mimetype: image/png + - name: file.txt + mimetype: text/plain + - name: index.txt + mimetype: text/plain + assert: + that: + - got[0]['name'] == exp[0]['name'] + - got[0]['mimetype'] == exp[0]['mimetype'] + - got[0]['children'][0]['name'] == exp[0]['children'][0]['name'] + - got[0]['children'][0]['mimetype'] == exp[0]['children'][0]['mimetype'] + - got[0]['children'][1]['name'] == exp[0]['children'][1]['name'] + - got[0]['children'][1]['mimetype'] == exp[0]['children'][1]['mimetype'] diff --git a/test-requirements.txt b/test-requirements.txt index 684b5a661..0a19d62d6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. flake8 +GitPython>=2.1.8,<2.1.12 zuul # We need to pin the ansible version directly here; per the diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index 3b964bb37..108152981 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -413,6 +413,14 @@ nodes: - secondary +- job: + name: zuul-jobs-test-generate-zuul-manifest + description: Test the generate-zuul-manifest role + run: test-playbooks/generate-zuul-manifest.yaml + files: + - ^roles/generate-zuul-manifest/.* + - ^test-playbooks/generate-zuul-manifest.yaml + - job: name: zuul-jobs-test-upload-git-mirror description: Test the upload-git-mirror role @@ -446,6 +454,7 @@ - zuul-jobs-test-multinode-roles-ubuntu-bionic - zuul-jobs-test-multinode-roles-ubuntu-trusty - zuul-jobs-test-multinode-roles-ubuntu-xenial + - zuul-jobs-test-generate-zuul-manifest - zuul-jobs-test-upload-git-mirror gate: jobs: *id001