Fix CORS and endpoint in AWS log upload

The AWS log upload module does not behave like the rest of the
modules: it does not set CORS headers, which means that logs will
not be usable in the Zuul web UI without additional manual
configuration.  The other uploaders all set CORS settings
automatically.

Correct this in the same way that we do for other clouds.

Additionally, if an S3 bucket is created outside of us-east-1, then
objects in it must be addressed using the bucket name as hostname,
rather than the bare host s3.amazonaws.com.  Update the returned
URL to use the bucket hostname.  This works as well for buckets in
us-east-1.

Also, fix the command line test invocation.

Change-Id: Id92f3cf506b442d0ee5932c6e9d931df19c2cc71
This commit is contained in:
James E. Blair 2022-09-21 07:12:31 -07:00
parent 9e22cfdb0f
commit 26bf1e516e

View File

@ -73,11 +73,13 @@ class Uploader():
if endpoint: if endpoint:
self.endpoint = endpoint self.endpoint = endpoint
self.url = os.path.join(endpoint,
bucket, self.prefix)
else: else:
self.endpoint = 'https://s3.amazonaws.com/' self.endpoint = 'https://s3.amazonaws.com/'
return_endpoint = 'https://' + bucket + '.s3.amazonaws.com/'
self.url = os.path.join(self.endpoint, self.url = os.path.join(return_endpoint,
bucket, self.prefix) self.prefix)
self.s3 = boto3.resource('s3', self.s3 = boto3.resource('s3',
endpoint_url=self.endpoint, endpoint_url=self.endpoint,
@ -85,6 +87,36 @@ class Uploader():
aws_secret_access_key=aws_secret_key) aws_secret_access_key=aws_secret_key)
self.bucket = self.s3.Bucket(bucket) self.bucket = self.s3.Bucket(bucket)
cors = {
'CORSRules': [{
'AllowedMethods': ['GET', 'HEAD'],
'AllowedOrigins': ['*'],
}]
}
client = boto3.client('s3',
endpoint_url=self.endpoint,
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key)
try:
current_cors = None
try:
current_cors = client.get_bucket_cors(Bucket=bucket)
except Exception as e:
# If the error is that we don't have any CORES rules,
# that's okay.
if 'NoSuchCORSConfiguration' not in str(e):
raise
if current_cors:
if current_cors['CORSRules'] != cors['CORSRules']:
current_cors = None
if not current_cors:
client.put_bucket_cors(Bucket=bucket,
CORSConfiguration=cors)
except Exception as e:
# MinIO (which we use in testing) doesn't implement this method
if 'MalformedXML' not in str(e):
raise
def upload(self, file_list): def upload(self, file_list):
"""Spin up thread pool to upload to storage""" """Spin up thread pool to upload to storage"""
@ -296,7 +328,7 @@ def cli_main():
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
logging.captureWarnings(True) logging.captureWarnings(True)
url = run(args.bucket, args.files, url = run(args.bucket, files=args.files,
prefix=args.prefix, prefix=args.prefix,
public=not args.no_public, public=not args.no_public,
endpoint=args.endpoint) endpoint=args.endpoint)