local-log-download : role with script to download all log files

This is an alternative to I98c80f657f38c5e1ed5f28e5d36988a3429ad1f8
which does not modify the upload script, but rather queries the API
and manifest for what to download.

The script is a hybrid of python and bash to not implement json API
parsing badly in bash, but not replicate curl badly in python either.
The script sanity checks for dependencies, which are considered pretty
standard for any developer who would be interested in downloading logs
like this.

The role writes out the script with the correct build values coded
into it, so a potential user just has to run it without any arguments
or modification.

Change-Id: Ic33732adbfd3210191bf4976c3ee316cfc50568e

doc/source/log-roles.rst

@@ -8,6 +8,7 @@ Log Roles
 .. zuul:autorole:: fetch-output-openshift
 .. zuul:autorole:: generate-zuul-manifest
 .. zuul:autorole:: htmlify-logs
+.. zuul:autorole:: local-log-download
 .. zuul:autorole:: merge-output-to-logs
 .. zuul:autorole:: publish-artifacts-to-fileserver
 .. zuul:autorole:: set-zuul-log-path-fact

roles/local-log-download/README.rst

@@ -0,0 +1,11 @@
+Add a script for users to bulk download logs locally
+
+This adds a script for users to bulk download all logs to their local
+system.  It queries the Zuul API for the manifest and then copies all
+files locally from the log server.
+
+**Role Variables**
+
+.. zuul:rolevar:: local_log_download_api
+
+   The Zuul API endpoint to use.  Example: ``https://zuul.example.org/api/tenant/{{ zuul.tenant }}``

roles/local-log-download/tasks/main.yaml

@@ -0,0 +1,12 @@
+- name: Check API endpoint is defined
+  assert:
+    that:
+      - local_log_download_api is defined
+    msg: 'local_log_download_api must be defined'
+
+- name: Create download script
+  delegate_to: localhost
+  template:
+    dest: '{{ zuul.executor.log_root }}/download-logs.sh'
+    src: 'download-logs.sh.j2'
+    mode: 0755

roles/local-log-download/templates/download-logs.sh.j2

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e
+
+ZUUL_API=${ZUUL_API:-"{{ local_log_download_api }}"}
+ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"}
+
+ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID}
+
+(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; }
+command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; }
+command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; }
+
+function log {
+    echo "$(date -Iseconds) | $@"
+}
+
+{#
+ # Parse the zuul build results to find the manifest, then parse
+ # the manifest and print files that shoud be downloaded.  The
+ # first line of output is the base url, then every line after is a
+ # file to download.
+#}
+function get_urls {
+    /usr/bin/env python3 - <<EOF
+import gzip
+import json
+import urllib.request
+
+base_url = urllib.request.urlopen("${ZUUL_API_URL}").read()
+base_json = json.loads(base_url)
+manifest_url = [x['url'] for x in base_json['artifacts'] if x.get('metadata', {}).get('type') == 'zuul_manifest'][0]
+manifest = urllib.request.urlopen(manifest_url)
+if manifest.info().get('Content-Encoding') == 'gzip':
+    manifest_json = json.loads(gzip.decompress(manifest.read()))
+else:
+    manifest_json = json.loads(manifest.read())
+
+def p(node, parent):
+    if node.get('mimetype') != 'application/directory':
+        print(parent+node['name'])
+    if node.get('children'):
+        for child in node.get('children'):
+                p(child, parent+node['name']+'/')
+
+print(base_json['log_url'])
+for i in manifest_json['tree']:
+    p(i, '')
+
+EOF
+}
+
+function save_file {
+    local base_url="$1"
+    local file="$2"
+
+    curl -s --compressed --create-dirs -o "${file}" "${base_url}/${file}"
+   {#
+    # Using --compressed we will send an Accept-Encoding: gzip header
+    # and the data will come to us across the network compressed.
+    # However, we see weird things like .gz files (as stored on disk)
+    # actually uncompressed, so we check if this really looks like an
+    # ASCII file and rename for clarity.
+   #}
+    if [[ "${file}" == *.gz ]]; then
+        local type=$(file "${file}")
+        if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then
+            local new_name=${file%.gz}
+            log "Renaming to ${new_name}"
+            mv "${file}" "${new_name}"
+        fi
+    fi
+
+}
+
+{#
+ # - read in file _files so we exit if the lookup fails for some reason
+ # - jinja gets confused with ${#files[@]} as it looks like a comment,
+ #   wrap it in raw
+#}
+log "Querying ${ZUUL_API_URL} for manifest"
+_files="$(get_urls)"
+readarray -t files <<< "${_files}"
+{% raw %}
+len="${#files[@]}"
+{% endraw %}
+if [[ -z "${DOWNLOAD_DIR}" ]]; then
+    DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX)
+fi
+log "Saving logs to ${DOWNLOAD_DIR}"
+
+pushd "${DOWNLOAD_DIR}" > /dev/null
+
+base_url="${files[0]}"
+log "Getting logs from ${base_url}"
+for (( i=1; i<$len; i++ )); do
+    file="${files[i]}"
+    printf -v _out "  %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 ))
+    log "$_out"
+    save_file $base_url $file
+done
+
+popd >/dev/null
+
+log "Download complete!"

test-playbooks/local-log-download.yaml

@@ -0,0 +1,21 @@
+- hosts: all
+  tasks:
+
+    - name: Run local-log-download role
+      include_role:
+        name: local-log-download
+      vars:
+        local_log_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}'
+
+  post_tasks:
+    - name: Check for download script
+      delegate_to: localhost
+      file:
+        path: "{{ zuul.executor.log_root }}/download-logs.sh"
+        state: file
+      register: download_script
+
+    - name: Validate download script
+      assert:
+        that:
+          - download_script is not changed

zuul-tests.d/logs-jobs.yaml

@@ -0,0 +1,18 @@
+- job:
+    name: zuul-jobs-test-local-log-download
+    description: Test the local-log-download role
+    files:
+      - roles/local-log-download/.*
+    run: test-playbooks/local-log-download.yaml
+
+# -* AUTOGENERATED *-
+#  The following project section is autogenerated by
+#    tox -e update-test-platforms
+#  Please re-run to generate new job lists
+
+- project:
+    check:
+      jobs: &id001
+        - zuul-jobs-test-local-log-download
+    gate:
+      jobs: *id001