diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst index 1ebcd3e4d..6653b869f 100644 --- a/doc/source/log-roles.rst +++ b/doc/source/log-roles.rst @@ -8,6 +8,7 @@ Log Roles .. zuul:autorole:: fetch-output-openshift .. zuul:autorole:: generate-zuul-manifest .. zuul:autorole:: htmlify-logs +.. zuul:autorole:: local-log-download .. zuul:autorole:: merge-output-to-logs .. zuul:autorole:: publish-artifacts-to-fileserver .. zuul:autorole:: set-zuul-log-path-fact diff --git a/roles/local-log-download/README.rst b/roles/local-log-download/README.rst new file mode 100644 index 000000000..7448fd3d4 --- /dev/null +++ b/roles/local-log-download/README.rst @@ -0,0 +1,11 @@ +Add a script for users to bulk download logs locally + +This adds a script for users to bulk download all logs to their local +system. It queries the Zuul API for the manifest and then copies all +files locally from the log server. + +**Role Variables** + +.. zuul:rolevar:: local_log_download_api + + The Zuul API endpoint to use. Example: ``https://zuul.example.org/api/tenant/{{ zuul.tenant }}`` diff --git a/roles/local-log-download/tasks/main.yaml b/roles/local-log-download/tasks/main.yaml new file mode 100644 index 000000000..0c3cb94b6 --- /dev/null +++ b/roles/local-log-download/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Check API endpoint is defined + assert: + that: + - local_log_download_api is defined + msg: 'local_log_download_api must be defined' + +- name: Create download script + delegate_to: localhost + template: + dest: '{{ zuul.executor.log_root }}/download-logs.sh' + src: 'download-logs.sh.j2' + mode: 0755 diff --git a/roles/local-log-download/templates/download-logs.sh.j2 b/roles/local-log-download/templates/download-logs.sh.j2 new file mode 100644 index 000000000..7047e1290 --- /dev/null +++ b/roles/local-log-download/templates/download-logs.sh.j2 @@ -0,0 +1,105 @@ +#!/bin/bash + +set -e + +ZUUL_API=${ZUUL_API:-"{{ local_log_download_api }}"} +ZUUL_BUILD_UUID=${ZUUL_BUILD_UUID:-"{{ zuul.build }}"} + +ZUUL_API_URL=${ZUUL_API}/build/${ZUUL_BUILD_UUID} + +(( ${BASH_VERSION%%.*} >= 4 )) || { echo >&2 "bash >=4 required to download."; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo >&2 "Python3 is required to download."; exit 1; } +command -v curl >/dev/null 2>&1 || { echo >&2 "curl is required to download."; exit 1; } + +function log { + echo "$(date -Iseconds) | $@" +} + +{# + # Parse the zuul build results to find the manifest, then parse + # the manifest and print files that shoud be downloaded. The + # first line of output is the base url, then every line after is a + # file to download. +#} +function get_urls { + /usr/bin/env python3 - < /dev/null + +base_url="${files[0]}" +log "Getting logs from ${base_url}" +for (( i=1; i<$len; i++ )); do + file="${files[i]}" + printf -v _out " %-80s [ %04d/%04d ]" "${file}" "${i}" $(( len -1 )) + log "$_out" + save_file $base_url $file +done + +popd >/dev/null + +log "Download complete!" diff --git a/test-playbooks/local-log-download.yaml b/test-playbooks/local-log-download.yaml new file mode 100644 index 000000000..2bf84e7cc --- /dev/null +++ b/test-playbooks/local-log-download.yaml @@ -0,0 +1,21 @@ +- hosts: all + tasks: + + - name: Run local-log-download role + include_role: + name: local-log-download + vars: + local_log_download_api: 'https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}' + + post_tasks: + - name: Check for download script + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/download-logs.sh" + state: file + register: download_script + + - name: Validate download script + assert: + that: + - download_script is not changed diff --git a/zuul-tests.d/logs-jobs.yaml b/zuul-tests.d/logs-jobs.yaml new file mode 100644 index 000000000..0160c8b78 --- /dev/null +++ b/zuul-tests.d/logs-jobs.yaml @@ -0,0 +1,18 @@ +- job: + name: zuul-jobs-test-local-log-download + description: Test the local-log-download role + files: + - roles/local-log-download/.* + run: test-playbooks/local-log-download.yaml + +# -* AUTOGENERATED *- +# The following project section is autogenerated by +# tox -e update-test-platforms +# Please re-run to generate new job lists + +- project: + check: + jobs: &id001 + - zuul-jobs-test-local-log-download + gate: + jobs: *id001