Add generate-zuul-manifest role

Also, pin gitpython since it's gone py3-only.

Change-Id: I0d626945908ec4df785aea793f6243c6fdfbdb14

doc/source/log-roles.rst

@@ -5,6 +5,7 @@ Log Roles
 .. zuul:autorole:: ara-report
 .. zuul:autorole:: ensure-output-dirs
 .. zuul:autorole:: fetch-output
+.. zuul:autorole:: generate-zuul-manifest
 .. zuul:autorole:: htmlify-logs
 .. zuul:autorole:: merge-output-to-logs
 .. zuul:autorole:: publish-artifacts-to-fileserver

roles/generate-zuul-manifest/README.rst

@@ -0,0 +1,28 @@
+Generate a Zuul manifest file for log uploading
+
+This generates a manifest file in preparation for uploading along
+with logs.  The Zuul web interface can fetch this file in order to
+display logs from a build.
+
+
+**Role Variables**
+
+.. zuul:rolevar:: generate_zuul_manifest_root
+   :default: {{ zuul.executor.log_dir }}
+
+   The root directory to index.
+
+.. zuul:rolevar:: generate_zuul_manifest_filename
+   :default: zuul-manifest.json
+
+   The name of the manifest file.
+
+.. zuul:rolevar:: generate_zuul_manifest_output
+   :default: {{ zuul.executor.log_dir }}/{{ generate_zuul_manifest_filename }}
+
+   The path to the output manifest file.
+
+.. zuul:rolevar:: generate_zuul_manifest_type
+   :default: zuul_manifest
+
+   The artifact type to return to Zuul.

roles/generate-zuul-manifest/defaults/main.yaml

@@ -0,0 +1,4 @@
+generate_zuul_manifest_root: "{{ zuul.executor.log_dir }}"
+generate_zuul_manifest_filename: "zuul-manifest.json"
+generate_zuul_manifest_output: "{{ zuul.executor.log_dir }}/{{ generate_zuul_manifest_filename }}"
+generate_zuul_manifest_type: "zuul_manifest"

roles/generate-zuul-manifest/library/generate_manifest.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import logging
+import mimetypes
+import os
+import stat
+import sys
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+mimetypes.init()
+
+
+def path_in_tree(root, path):
+    full_path = os.path.realpath(os.path.abspath(
+        os.path.expanduser(path)))
+    if not full_path.startswith(root):
+        logging.debug("Skipping path outside root: %s" % (path,))
+        return False
+    return True
+
+
+def walk(root, original_root=None):
+    if original_root is None:
+        original_root = root
+    logging.debug("Walk: %s", root)
+    data = []
+    dirs = []
+    files = []
+    for e in os.listdir(root):
+        if os.path.isdir(os.path.join(root, e)):
+            if not os.path.islink(os.path.join(root, e)):
+                dirs.append(e)
+        else:
+            files.append(e)
+    for d in sorted(dirs):
+        logging.debug("Directory: %s", d)
+        path = os.path.join(root, d)
+        if not path_in_tree(original_root, path):
+            continue
+        data.append(dict(name=d,
+                         mimetype='application/directory',
+                         encoding=None,
+                         children=walk(os.path.join(root, d), original_root)))
+    for f in sorted(files):
+        logging.debug("File: %s", f)
+        path = os.path.join(root, f)
+        if not path_in_tree(original_root, path):
+            continue
+        mime_guess, encoding = mimetypes.guess_type(path)
+        if not mime_guess:
+            mime_guess = 'text/plain'
+        st = os.stat(path)
+        last_modified = st[stat.ST_MTIME]
+        size = st[stat.ST_SIZE]
+        data.append(dict(name=f,
+                         mimetype=mime_guess,
+                         encoding=encoding,
+                         last_modified=last_modified,
+                         size=size))
+    return data
+
+
+def run(root_path, output):
+    data = walk(root_path, root_path)
+    with open(output, 'w') as f:
+        f.write(json.dumps({'tree': data}))
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            root=dict(type='path'),
+            output=dict(type='path'),
+        )
+    )
+
+    p = module.params
+    run(p.get('root'), p.get('output'))
+
+    module.exit_json(changed=True)
+
+
+def cli_main():
+    parser = argparse.ArgumentParser(
+        description="Generate a Zuul file manifest"
+    )
+    parser.add_argument('--verbose', action='store_true',
+                        help='show debug information')
+    parser.add_argument('root',
+                        help='Root of upload directory')
+    parser.add_argument('output',
+                        help='Output file path')
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    run(args.root, args.output)
+
+
+if __name__ == '__main__':
+    if sys.stdin.isatty():
+        cli_main()
+    else:
+        ansible_main()

roles/generate-zuul-manifest/library/test-fixtures/links/job-output.json

@@ -0,0 +1 @@
+{"test": "foo"}

roles/generate-zuul-manifest/library/test-fixtures/logs/controller/cpu-load.svg

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg>
+</svg>

roles/generate-zuul-manifest/library/test-fixtures/logs/job-output.json

@@ -0,0 +1 @@
+{"test": "foo"}

roles/generate-zuul-manifest/library/test_generate_manifest.py

@@ -0,0 +1,128 @@
+# Copyright (C) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import testtools
+import fixtures
+
+from .generate_manifest import walk
+
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+                           'test-fixtures')
+
+
+class SymlinkFixture(fixtures.Fixture):
+    links = [
+        ('bad_symlink', '/etc'),
+        ('bad_symlink_file', '/etc/issue'),
+        ('good_symlink', 'controller'),
+        ('recursive_symlink', '.'),
+        ('symlink_file', 'job-output.json'),
+        ('symlink_loop_a', 'symlink_loop'),
+        ('symlink_loop/symlink_loop_b', '..'),
+    ]
+
+    def _setUp(self):
+        self._cleanup()
+        for (src, target) in self.links:
+            path = os.path.join(FIXTURE_DIR, 'links', src)
+            os.symlink(target, path)
+        self.addCleanup(self._cleanup)
+
+    def _cleanup(self):
+        for (src, target) in self.links:
+            path = os.path.join(FIXTURE_DIR, 'links', src)
+            if os.path.exists(path):
+                os.unlink(path)
+
+
+class TestFileList(testtools.TestCase):
+
+    def flatten(self, result, out=None, path=''):
+        if out is None:
+            out = []
+        dirs = []
+        for x in result:
+            x['_relative_path'] = os.path.join(path, x['name'])
+            out.append(x)
+            if 'children' in x:
+                dirs.append(x)
+        for x in dirs:
+            self.flatten(x['children'], out, x['_relative_path'])
+            x.pop('children')
+        return out
+
+    def assert_files(self, root, result, files):
+        self.assertEqual(len(result), len(files))
+        for expected, received in zip(files, result):
+            self.assertEqual(expected[0], received['_relative_path'])
+            if expected[0] and expected[0][-1] == '/':
+                efilename = os.path.split(
+                    os.path.dirname(expected[0]))[1] + '/'
+            else:
+                efilename = os.path.split(expected[0])[1]
+            self.assertEqual(efilename, received['name'])
+            full_path = os.path.join(root, received['_relative_path'])
+            if received['mimetype'] == 'application/directory':
+                self.assertTrue(os.path.isdir(full_path))
+            else:
+                self.assertTrue(os.path.isfile(full_path))
+            self.assertEqual(expected[1], received['mimetype'])
+            self.assertEqual(expected[2], received['encoding'])
+
+    def find_file(self, file_list, path):
+        for f in file_list:
+            if f.relative_path == path:
+                return f
+
+    def test_single_dir(self):
+        '''Test a single directory with a trailing slash'''
+
+        root = os.path.join(FIXTURE_DIR, 'logs')
+        fl = walk(root)
+        self.assert_files(root, self.flatten(fl), [
+            ('controller', 'application/directory', None),
+            ('zuul-info', 'application/directory', None),
+            ('job-output.json', 'application/json', None),
+            ('controller/subdir', 'application/directory', None),
+            ('controller/compressed.gz', 'text/plain', 'gzip'),
+            ('controller/cpu-load.svg', 'image/svg+xml', None),
+            ('controller/journal.xz', 'text/plain', 'xz'),
+            ('controller/service_log.txt', 'text/plain', None),
+            ('controller/syslog', 'text/plain', None),
+            ('controller/subdir/subdir.txt', 'text/plain', None),
+            ('zuul-info/inventory.yaml', 'text/plain', None),
+            ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+        ])
+
+    def test_symlinks(self):
+        '''Test symlinks'''
+        self.useFixture(SymlinkFixture())
+        root = os.path.join(FIXTURE_DIR, 'links')
+        fl = walk(root)
+        self.assert_files(root, self.flatten(fl), [
+            ('controller', 'application/directory', None),
+            ('symlink_loop', 'application/directory', None),
+            ('job-output.json', 'application/json', None),
+            ('symlink_file', 'text/plain', None),
+            ('controller/service_log.txt', 'text/plain', None),
+            ('symlink_loop/placeholder', 'text/plain', None),
+        ])

roles/generate-zuul-manifest/tasks/main.yaml

@@ -0,0 +1,14 @@
+- name: Generate Zuul manifest
+  generate_manifest:
+    root: "{{ generate_zuul_manifest_root }}"
+    output: "{{ generate_zuul_manifest_output }}"
+
+- name: Return Zuul manifest URL to Zuul
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: Manifest
+            url: "{{ generate_zuul_manifest_filename }}"
+            metadata:
+              type: "{{ generate_zuul_manifest_type }}"

test-playbooks/generate-zuul-manifest.yaml

@@ -0,0 +1,60 @@
+- name: Run tests for the generate-zuul-manifest role
+  hosts: all
+  pre_tasks:
+    - name: Create test directories
+      file:
+        path: "{{ ansible_user_dir }}/{{ item }}"
+        state: directory
+      loop:
+        - tests
+        - tests/logs
+
+    - name: Create tests files
+      copy:
+        dest: "{{ ansible_user_dir }}/{{ item }}"
+        content: ""
+      loop:
+        - tests/index.txt
+        - tests/logs/file.txt
+        - tests/logs/file.png
+
+  roles:
+    - role: generate-zuul-manifest
+      generate_zuul_manifest_root: "{{ ansible_user_dir }}/tests"
+      generate_zuul_manifest_filename: "test-manifest.json"
+      generate_zuul_manifest_output: "{{ ansible_user_dir }}/tests/{{ generate_zuul_manifest_filename }}"
+      generate_zuul_manifest_type: "test_zuul_manifest"
+
+  post_tasks:
+    - name: Fetch output
+      fetch:
+        src: "{{ ansible_user_dir }}/tests/test-manifest.json"
+        flat: true
+        dest: "{{ zuul.executor.log_root }}/"
+
+    - name: Load output
+      include_vars:
+        file: "{{ zuul.executor.log_root }}/test-manifest.json"
+        name: manifest
+
+    - name: Check output
+      vars:
+        got: "{{ manifest['tree'] }}"
+        exp:
+          - name: logs
+            mimetype: application/directory
+            children:
+              - name: file.png
+                mimetype: image/png
+              - name: file.txt
+                mimetype: text/plain
+          - name: index.txt
+            mimetype: text/plain
+      assert:
+        that:
+          - got[0]['name'] == exp[0]['name']
+          - got[0]['mimetype'] == exp[0]['mimetype']
+          - got[0]['children'][0]['name'] == exp[0]['children'][0]['name']
+          - got[0]['children'][0]['mimetype'] == exp[0]['children'][0]['mimetype']
+          - got[0]['children'][1]['name'] == exp[0]['children'][1]['name']
+          - got[0]['children'][1]['mimetype'] == exp[0]['children'][1]['mimetype']

test-requirements.txt

@@ -2,6 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 flake8
+GitPython>=2.1.8,<2.1.12
 zuul
 
 # We need to pin the ansible version directly here; per the

zuul-tests.d/general-roles-jobs.yaml

@@ -413,6 +413,14 @@
           nodes:
             - secondary
 
+- job:
+    name: zuul-jobs-test-generate-zuul-manifest
+    description: Test the generate-zuul-manifest role
+    run: test-playbooks/generate-zuul-manifest.yaml
+    files:
+      - ^roles/generate-zuul-manifest/.*
+      - ^test-playbooks/generate-zuul-manifest.yaml
+
 - job:
     name: zuul-jobs-test-upload-git-mirror
     description: Test the upload-git-mirror role
@@ -446,6 +454,7 @@
         - zuul-jobs-test-multinode-roles-ubuntu-bionic
         - zuul-jobs-test-multinode-roles-ubuntu-trusty
         - zuul-jobs-test-multinode-roles-ubuntu-xenial
+        - zuul-jobs-test-generate-zuul-manifest
         - zuul-jobs-test-upload-git-mirror
     gate:
       jobs: *id001