zuul/zuul-jobs
Code Issues Proposed changes

Merge "Revert "upload-logs-swift: Create a download script""

Browse Source
This commit is contained in:
Zuul 2020-03-27 14:26:24 +00:00 committed by Gerrit Code Review
commit 753305bd7d
6 changed files with 9 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
roles/upload-logs-swift/README.rst
View File

@ -76,17 +76,3 @@ This uploads logs to an OpenStack Object Store (Swift) container.
More details can be found at
:zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
.. zuul:rolevar:: zuul_log_include_download_script
:default: False
Generate a script from ``zuul_log_download_template`` in the root
directory of the uploaded logs to facilitate easy bulk download.
.. zuul:rolevar:: zuul_log_download_template
:default: templates/download-logs.sh.j2
Path to template file if ``zuul_log_include_download_script`` is
set. See the sample file for parameters available to the template.
The file will be placed in the root of the uploaded logs (with
``.j2`` suffix removed).

2
roles/upload-logs-swift/defaults/main.yaml
View File

@ -2,5 +2,3 @@ zuul_log_partition: false
zuul_log_container: logs
zuul_log_container_public: true
zuul_log_create_indexes: true
zuul_log_include_download_script: false
zuul_log_download_template: '{{ role_path }}/templates/download-logs.sh.j2'

87
roles/upload-logs-swift/library/test-fixtures/download-logs-sample.sh
View File

@ -1,87 +0,0 @@
#!/bin/bash
# Download all logs
#
# To use this file
#
# curl "http://fakebaseurl.com/download-logs.sh" | bash
#
# Logs will be copied in a temporary directory as described in the
# output. Set DOWNLOAD_DIR to an empty directory if you wish to
# override this.
#
BASE_URL=http://fakebaseurl.com
function log {
echo "$(date -Iseconds) | $@"
}
function save_file {
local file="$1"
curl -s --compressed --create-dirs -o "${file}" "${BASE_URL}/${file}"
# Using --compressed we will send an Accept-Encoding: gzip header
# and the data will come to us across the network compressed.
# However, sometimes things like OpenStack's log server will send
# .gz files (as stored on its disk) uncompressed, so we check if
# this really looks like an ASCII file and rename for clarity.
if [[ "${file}" == *.gz ]]; then
local type=$(file "${file}")
if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then
local new_name=${file%.gz}
log "Renaming to ${new_name}"
mv "${file}" "${new_name}"
fi
fi
}
if [[ -z "${DOWNLOAD_DIR}" ]]; then
DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX)
fi
log "Saving logs to ${DOWNLOAD_DIR}"
pushd "${DOWNLOAD_DIR}" > /dev/null
log "Getting ${BASE_URL}/job-output.json [ 0001 / 0011 ]"
save_file "job-output.json"
log "Getting ${BASE_URL}/Ꮓບບξ-unicode.txt [ 0002 / 0011 ]"
save_file "Ꮓບບξ-unicode.txt"
log "Getting ${BASE_URL}/controller/compressed.gz [ 0003 / 0011 ]"
save_file "controller/compressed.gz"
log "Getting ${BASE_URL}/controller/cpu-load.svg [ 0004 / 0011 ]"
save_file "controller/cpu-load.svg"
log "Getting ${BASE_URL}/controller/journal.xz [ 0005 / 0011 ]"
save_file "controller/journal.xz"
log "Getting ${BASE_URL}/controller/service_log.txt [ 0006 / 0011 ]"
save_file "controller/service_log.txt"
log "Getting ${BASE_URL}/controller/syslog [ 0007 / 0011 ]"
save_file "controller/syslog"
log "Getting ${BASE_URL}/controller/subdir/foo::3.txt [ 0008 / 0011 ]"
save_file "controller/subdir/foo::3.txt"
log "Getting ${BASE_URL}/controller/subdir/subdir.txt [ 0009 / 0011 ]"
save_file "controller/subdir/subdir.txt"
log "Getting ${BASE_URL}/zuul-info/inventory.yaml [ 0010 / 0011 ]"
save_file "zuul-info/inventory.yaml"
log "Getting ${BASE_URL}/zuul-info/zuul-info.controller.txt [ 0011 / 0011 ]"
save_file "zuul-info/zuul-info.controller.txt"
popd >/dev/null
log "Download complete!"

72
roles/upload-logs-swift/library/zuul_swift_upload.py
View File

@ -30,7 +30,6 @@ import io
import logging
import mimetypes
import os
import jinja2
try:
import queue as queuelib
except ImportError:
@ -55,7 +54,6 @@ import requests.exceptions
import requestsexceptions
import keystoneauth1.exceptions
from ansible.module_utils._text import to_text
from ansible.module_utils.basic import AnsibleModule
try:
@ -267,15 +265,13 @@ class FileDetail():
to push to swift.
"""
def __init__(self, full_path, relative_path,
filename=None, is_index=False):
def __init__(self, full_path, relative_path, filename=None):
"""
Args:
full_path (str): The absolute path to the file on disk.
relative_path (str): The relative path from the artifacts source
used for links.
filename (str): An optional alternate filename in links.
is_index (bool): Is this file an index
"""
# Make FileNotFoundError exception to be compatible with python2
try:
@ -289,7 +285,6 @@ class FileDetail():
else:
self.filename = filename
self.relative_path = relative_path
self.is_index = is_index
if self.full_path and os.path.isfile(self.full_path):
mime_guess, encoding = mimetypes.guess_type(self.full_path)
@ -310,8 +305,7 @@ class FileDetail():
def __repr__(self):
t = 'Folder' if self.folder else 'File'
return '<%s %s%s>' % (t, self.relative_path,
' (index)' if self.is_index else '')
return '<%s %s>' % (t, self.relative_path)
class FileList(Sequence):
@ -417,7 +411,6 @@ class Indexer():
FileList
- make_indexes() : make index.html in folders
- make_download_script() : make a script to download all logs
"""
def __init__(self, file_list):
'''
@ -537,8 +530,7 @@ class Indexer():
if full_path:
filename = os.path.basename(full_path)
relative_name = os.path.join(folder, filename)
indexes[folder] = FileDetail(full_path, relative_name,
is_index=True)
indexes[folder] = FileDetail(full_path, relative_name)
# This appends the index file at the end of the group of files
# for each directory.
@ -561,41 +553,6 @@ class Indexer():
new_list.reverse()
self.file_list.file_list = new_list
def make_download_script(self, base_url, download_template):
'''Make a download script from template
Note since you need the base_url, it really only makes sense
to call this after the Uploader() is initalised.
Args:
base_url (str): The base URL to prefix
download_template (str): Path to a jinja2 template
Return:
None; a file with the same name as the template (stripped of
.j2 if present) is added to self.file_list for upload.
'''
# Prune the list to just be files, no indexes (this should run
# before indexing anyway)
download_files = [f for f in self.file_list
if not f.folder and not f.is_index]
output_filename = os.path.basename(download_template[:-3]
if download_template.endswith('.j2')
else download_template)
output = os.path.join(self.file_list.get_tempdir(), output_filename)
with open(download_template) as f, open(output, 'wb') as output:
logging.debug("Writing template %s" % output.name)
template = jinja2.Template(f.read())
rendered = template.stream(
base_url=base_url.rstrip('/'),
# jinja wants unicode input
file_list=[to_text(f.relative_path) for f in download_files])
rendered.dump(output, encoding='utf-8')
download_script = FileDetail(output.name, output_filename)
self.file_list.file_list.append(download_script)
class GzipFilter():
chunk_size = 16384
@ -813,7 +770,7 @@ class Uploader():
def run(cloud, container, files,
indexes=True, parent_links=True, topdir_parent_link=False,
partition=False, footer='index_footer.html', delete_after=15552000,
prefix=None, public=True, dry_run=False, download_template=''):
prefix=None, public=True, dry_run=False):
if prefix:
prefix = prefix.lstrip('/')
@ -829,16 +786,8 @@ def run(cloud, container, files,
for file_path in files:
file_list.add(file_path)
# Upload.
uploader = Uploader(cloud, container, prefix, delete_after,
public, dry_run)
indexer = Indexer(file_list)
# (Possibly) make download script
if download_template:
indexer.make_download_script(uploader.url, download_template)
# (Possibly) make indexes.
if indexes:
indexer.make_indexes(create_parent_links=parent_links,
@ -849,6 +798,9 @@ def run(cloud, container, files,
for x in file_list:
logging.debug(x)
# Upload.
uploader = Uploader(cloud, container, prefix, delete_after,
public, dry_run)
uploader.upload(file_list)
return uploader.url
@ -867,7 +819,6 @@ def ansible_main():
footer=dict(type='str'),
delete_after=dict(type='int'),
prefix=dict(type='str'),
download_template=dict(type='str'),
)
)
@ -882,8 +833,7 @@ def ansible_main():
footer=p.get('footer'),
delete_after=p.get('delete_after', 15552000),
prefix=p.get('prefix'),
public=p.get('public'),
download_template=p.get('download_template'))
public=p.get('public'))
except (keystoneauth1.exceptions.http.HttpError,
requests.exceptions.RequestException):
s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
@ -925,9 +875,6 @@ def cli_main():
'upload. Default is 6 months (15552000 seconds) '
'and if set to 0 X-Delete-After will not be set',
type=int)
parser.add_argument('--download-template', default='',
help='Path to a Jinja2 template that will be filled '
'out to create an automatic download script')
parser.add_argument('--prefix',
help='Prepend this path to the object names when '
'uploading')
@ -965,8 +912,7 @@ def cli_main():
delete_after=args.delete_after,
prefix=args.prefix,
public=not args.no_public,
dry_run=args.dry_run,
download_template=args.download_template)
dry_run=args.dry_run)
print(url)

25
roles/upload-logs-swift/tasks/main.yaml
View File

@ -16,12 +16,6 @@
tags:
- skip_ansible_lint
- name: Set download template
set_fact:
download_template: "{{ zuul_log_download_template }}"
when:
- zuul_log_include_download_script
- name: Upload logs to swift
delegate_to: localhost
zuul_swift_upload:
@ -34,27 +28,8 @@
files:
- "{{ zuul.executor.log_root }}/"
delete_after: "{{ zuul_log_delete_after | default(omit) }}"
download_template: "{{ download_template | default(omit) }}"
register: upload_results
- name: Get download script name
set_fact:
download_script: "{{ zuul_log_download_template | basename | regex_replace('\\.j2$') }}"
when:
- zuul_log_include_download_script
- name: Set download template artifact
zuul_return:
data:
zuul:
artifacts:
- name: Download all logs
url: '{{ download_script }}'
metadata:
command: 'curl "{{ upload_results.url }}/{{ download_script }}" | bash'
when:
- zuul_log_include_download_script
- name: Return log URL to Zuul
delegate_to: localhost
zuul_return:

57
roles/upload-logs-swift/templates/download-logs.sh.j2
View File

@ -1,57 +0,0 @@
#!/bin/bash
# Download all logs
#
# To use this file
#
# curl "{{ base_url }}/download-logs.sh" | bash
#
# Logs will be copied in a temporary directory as described in the
# output. Set DOWNLOAD_DIR to an empty directory if you wish to
# override this.
#
BASE_URL={{ base_url }}
function log {
echo "$(date -Iseconds) | $@"
}
function save_file {
local file="$1"
curl -s --compressed --create-dirs -o "${file}" "${BASE_URL}/${file}"
# Using --compressed we will send an Accept-Encoding: gzip header
# and the data will come to us across the network compressed.
# However, sometimes things like OpenStack's log server will send
# .gz files (as stored on its disk) uncompressed, so we check if
# this really looks like an ASCII file and rename for clarity.
if [[ "${file}" == *.gz ]]; then
local type=$(file "${file}")
if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then
local new_name=${file%.gz}
log "Renaming to ${new_name}"
mv "${file}" "${new_name}"
fi
fi
}
if [[ -z "${DOWNLOAD_DIR}" ]]; then
DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX)
fi
log "Saving logs to ${DOWNLOAD_DIR}"
pushd "${DOWNLOAD_DIR}" > /dev/null
{% set total_files = file_list | length %}
{% for file in file_list %}
log "Getting ${BASE_URL}/{{ '%-80s'|format(file) }} [ {{ '%04d'|format(loop.index) }} / {{ '%04d'|format(total_files) }} ]"
save_file "{{ file }}"
{% endfor %}
popd >/dev/null
log "Download complete!"