zuul/zuul-jobs
Code Issues Proposed changes

Also compress json files

Browse Source 
Json files typically get the mimetime application/json and thus are
not a sub type of text. However we want to compress them too.

Change-Id: I4f0532c6fe595dcebe2c21d8634691a2b1ceb6ed
This commit is contained in:
Tobias Henkel 2018-07-30 14:46:25 +02:00 committed by James E. Blair
parent 978eb46abe
commit cd68d14a2f
3 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
roles/wip-upload-logs-swift/library/test-fixtures/logs/job-output.json Normal file
View File

@ -0,0 +1 @@
{"test": "foo"}

8
roles/wip-upload-logs-swift/library/test_zuul_swift_upload.py
View File

@ -57,6 +57,7 @@ class TestFileList(unittest.TestCase):
('', 'application/directory', None),
('controller', 'application/directory', None),
('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None),
('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'),
@ -76,6 +77,7 @@ class TestFileList(unittest.TestCase):
('logs', 'application/directory', None),
('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None),
('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/journal.xz', 'text/plain', 'xz'),
@ -107,9 +109,10 @@ class TestFileList(unittest.TestCase):
('', 'application/directory', None),
('index.html', 'text/html', None),
('logs', 'application/directory', None),
('logs/index.html', 'text/html', None),
('logs/controller', 'application/directory', None),
('logs/zuul-info', 'application/directory', None),
('logs/job-output.json', 'application/json', None),
('logs/index.html', 'text/html', None),
('logs/controller/subdir', 'application/directory', None),
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
('logs/controller/journal.xz', 'text/plain', 'xz'),
@ -154,9 +157,10 @@ class TestFileList(unittest.TestCase):
self.assert_files(fl, [
('', 'application/directory', None),
('index.html', 'text/html', None),
('controller', 'application/directory', None),
('zuul-info', 'application/directory', None),
('job-output.json', 'application/json', None),
('index.html', 'text/html', None),
('controller/subdir', 'application/directory', None),
('controller/compressed.gz', 'text/plain', 'gzip'),
('controller/journal.xz', 'text/plain', 'xz'),

17
roles/wip-upload-logs-swift/library/zuul_swift_upload.py
View File

@ -392,6 +392,21 @@ class Uploader(object):
# No more work to do
return
@staticmethod
def _is_text_type(mimetype):
# We want to compress all text types.
if mimetype.startswith('text/'):
return True
# Further compress types that typically contain text but are no
# text sub type.
compress_types = [
'application/json',
]
if mimetype in compress_types:
return True
return False
def _post_file(self, fd):
relative_path = os.path.join(self.prefix, fd.relative_path)
headers = {}
@ -404,7 +419,7 @@ class Uploader(object):
for attempt in range(3):
try:
if not fd.folder:
if fd.encoding is None and fd.mimetype.startswith('text/'):
if fd.encoding is None and self._is_text_type(fd.mimetype):
headers['content-encoding'] = 'deflate'
data = DeflateFilter(open(fd.full_path, 'rb'))
else: