diff --git a/roles/upload-logs-base/library/zuul_google_storage_upload.py b/roles/upload-logs-base/library/zuul_google_storage_upload.py index 0536a9cc9..afca58771 100755 --- a/roles/upload-logs-base/library/zuul_google_storage_upload.py +++ b/roles/upload-logs-base/library/zuul_google_storage_upload.py @@ -93,7 +93,8 @@ class Uploader(): self.dry_run = dry_run if dry_run: - self.url = 'http://dry-run-url.com/a/path/' + self.endpoint = 'http://dry-run-url.com' + self.path = '/a/path/' return self.client = client @@ -107,9 +108,8 @@ class Uploader(): self.bucket.cors = cors self.bucket.website = {"mainPageSuffix": "index.html"} self.bucket.update() - - self.url = os.path.join('https://storage.googleapis.com/', - container, self.prefix) + self.endpoint = 'https://storage.googleapis.com' + self.path = os.path.join(container, self.prefix) def upload(self, file_list): """Spin up thread pool to upload to storage""" @@ -240,7 +240,7 @@ def run(container, files, # Upload. uploader = Uploader(client, container, prefix, dry_run) uploader.upload(file_list) - return uploader.url + return uploader.endpoint, uploader.path def ansible_main(): @@ -260,17 +260,18 @@ def ansible_main(): ) p = module.params - url = run(p.get('container'), p.get('files'), - indexes=p.get('indexes'), - parent_links=p.get('parent_links'), - topdir_parent_link=p.get('topdir_parent_link'), - partition=p.get('partition'), - footer=p.get('footer'), - prefix=p.get('prefix'), - credentials_file=p.get('credentials_file'), - project=p.get('project')) + endpoint, path = run(p.get('container'), p.get('files'), + indexes=p.get('indexes'), + parent_links=p.get('parent_links'), + topdir_parent_link=p.get('topdir_parent_link'), + partition=p.get('partition'), + footer=p.get('footer'), + prefix=p.get('prefix'), + credentials_file=p.get('credentials_file'), + project=p.get('project')) module.exit_json(changed=True, - url=url) + endpoint=endpoint, + path=path) def cli_main(): @@ -320,17 +321,17 @@ def cli_main(): if append_footer.lower() == 'none': append_footer = None - url = run(args.container, args.files, - indexes=not args.no_indexes, - parent_links=not args.no_parent_links, - topdir_parent_link=args.create_topdir_parent_link, - partition=args.partition, - footer=append_footer, - prefix=args.prefix, - dry_run=args.dry_run, - credentials_file=args.credentials_file, - project=args.project) - print(url) + _, path = run(args.container, args.files, + indexes=not args.no_indexes, + parent_links=not args.no_parent_links, + topdir_parent_link=args.create_topdir_parent_link, + partition=args.partition, + footer=append_footer, + prefix=args.prefix, + dry_run=args.dry_run, + credentials_file=args.credentials_file, + project=args.project) + print(path) if __name__ == '__main__': diff --git a/roles/upload-logs-base/library/zuul_s3_upload.py b/roles/upload-logs-base/library/zuul_s3_upload.py index 6a8d7964c..6192378ba 100755 --- a/roles/upload-logs-base/library/zuul_s3_upload.py +++ b/roles/upload-logs-base/library/zuul_s3_upload.py @@ -63,7 +63,8 @@ class Uploader(): self.dry_run = dry_run self.public = public if dry_run: - self.url = 'http://dry-run-url.com/a/path/' + self.endpoint = 'http://dry-run-url.com' + self.path = '/a/path' return self.prefix = prefix or '' @@ -73,8 +74,7 @@ class Uploader(): else: self.endpoint = 'https://s3.amazonaws.com/' - self.url = os.path.join(self.endpoint, - bucket, self.prefix) + self.path = os.path.join(bucket, self.prefix) self.s3 = boto3.resource('s3', endpoint_url=self.endpoint, @@ -223,7 +223,7 @@ def run(bucket, public, files, endpoint=None, aws_secret_key=aws_secret_key) upload_failures = uploader.upload(file_list) - return uploader.url, upload_failures + return uploader.endpoint, uploader.path, upload_failures def ansible_main(): @@ -245,24 +245,28 @@ def ansible_main(): ) p = module.params - url, failures = run(p.get('bucket'), - p.get('public'), - p.get('files'), - p.get('endpoint'), - indexes=p.get('indexes'), - parent_links=p.get('parent_links'), - topdir_parent_link=p.get('topdir_parent_link'), - partition=p.get('partition'), - footer=p.get('footer'), - prefix=p.get('prefix'), - aws_access_key=p.get('aws_access_key'), - aws_secret_key=p.get('aws_secret_key')) + endpoint, path, failures = run( + p.get('bucket'), + p.get('public'), + p.get('files'), + p.get('endpoint'), + indexes=p.get('indexes'), + parent_links=p.get('parent_links'), + topdir_parent_link=p.get('topdir_parent_link'), + partition=p.get('partition'), + footer=p.get('footer'), + prefix=p.get('prefix'), + aws_access_key=p.get('aws_access_key'), + aws_secret_key=p.get('aws_secret_key'), + ) if failures: module.fail_json(changed=True, - url=url, + endpoint=endpoint, + path=path, failures=failures) module.exit_json(changed=True, - url=url, + endpoint=endpoint, + path=path, failures=failures) @@ -290,11 +294,10 @@ def cli_main(): logging.basicConfig(level=logging.DEBUG) logging.captureWarnings(True) - url = run(args.bucket, args.files, - prefix=args.prefix, - public=not args.no_public, - endpoint=args.endpoint) - print(url) + _, path, _ = run(args.bucket, not args.no_public, args.files, + prefix=args.prefix, + endpoint=args.endpoint) + print(path) if __name__ == '__main__': diff --git a/roles/upload-logs-base/library/zuul_swift_upload.py b/roles/upload-logs-base/library/zuul_swift_upload.py index bfa34bcaa..9c7108064 100755 --- a/roles/upload-logs-base/library/zuul_swift_upload.py +++ b/roles/upload-logs-base/library/zuul_swift_upload.py @@ -77,7 +77,8 @@ class Uploader(): self.dry_run = dry_run if dry_run: - self.url = 'http://dry-run-url.com/a/path/' + self.endpoint = 'http://dry-run-url.com' + self.path = '/a/path' return self.cloud = cloud @@ -140,8 +141,8 @@ class Uploader(): else: endpoint = self.cloud.object_store.get_endpoint() container = os.path.join(endpoint, self.container) - - self.url = os.path.join(container, self.prefix) + self.endpoint = endpoint + self.path = os.path.join(self.container, self.prefix) def upload(self, file_list): """Spin up thread pool to upload to swift""" @@ -288,7 +289,7 @@ def run(cloud, container, files, uploader = Uploader(cloud, container, prefix, delete_after, public, dry_run) upload_failures = uploader.upload(file_list) - return uploader.url, upload_failures + return uploader.endpoint, uploader.path, upload_failures def ansible_main(): @@ -311,7 +312,7 @@ def ansible_main(): p = module.params cloud = get_cloud(p.get('cloud')) try: - url, upload_failures = run( + endpoint, path, upload_failures = run( cloud, p.get('container'), p.get('files'), indexes=p.get('indexes'), parent_links=p.get('parent_links'), @@ -334,7 +335,8 @@ def ansible_main(): region_name=cloud.config.region_name) module.exit_json( changed=True, - url=url, + endpoint=endpoint, + path=path, upload_failures=upload_failures, ) @@ -394,7 +396,7 @@ def cli_main(): if append_footer.lower() == 'none': append_footer = None - url, _ = run( + _, path, _ = run( get_cloud(args.cloud), args.container, args.files, indexes=not args.no_indexes, parent_links=not args.no_parent_links, @@ -406,7 +408,7 @@ def cli_main(): public=not args.no_public, dry_run=args.dry_run ) - print(url) + print(path) if __name__ == '__main__': diff --git a/roles/upload-logs-gcs/README.rst b/roles/upload-logs-gcs/README.rst index bc9dea17c..66ab5fe87 100644 --- a/roles/upload-logs-gcs/README.rst +++ b/roles/upload-logs-gcs/README.rst @@ -78,3 +78,10 @@ Google Cloud Application Default Credentials. :zuul:rolevar:`upload-logs-gcs.zuul_log_credentials_file`, the name of the Google Cloud project of the log container must also be supplied. + +.. zuul:rolevar:: zuul_log_storage_proxy_url + + The url of the proxy for the cloud object store. + If you are using zuul-storage-proxy to proxy requests for logs, set this + to the the URL of the log proxy server. When set, this role will replace + the cloud storage endpoint with this value in the log URL returned to Zuul. diff --git a/roles/upload-logs-gcs/tasks/main.yaml b/roles/upload-logs-gcs/tasks/main.yaml index 594f8431c..f880190df 100644 --- a/roles/upload-logs-gcs/tasks/main.yaml +++ b/roles/upload-logs-gcs/tasks/main.yaml @@ -34,5 +34,5 @@ zuul_return: data: zuul: - log_url: "{{ upload_results.url }}/" + log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}" when: upload_results is defined diff --git a/roles/upload-logs-s3/README.rst b/roles/upload-logs-s3/README.rst index 925bff7c1..76ad1d62a 100644 --- a/roles/upload-logs-s3/README.rst +++ b/roles/upload-logs-s3/README.rst @@ -76,3 +76,10 @@ installed in the Ansible environment on the Zuul executor. The endpoint to use when uploading logs to an s3 compatible service. By default this will be automatically constructed by boto but should be set when working with non-aws hosted s3 service. + +.. zuul:rolevar:: zuul_log_storage_proxy_url + + The url of the proxy for the cloud object store. + If you are using zuul-storage-proxy to proxy requests for logs, set this + to the the URL of the log proxy server. When set, this role will replace + the cloud storage endpoint with this value in the log URL returned to Zuul. diff --git a/roles/upload-logs-s3/tasks/main.yaml b/roles/upload-logs-s3/tasks/main.yaml index 385b9e2e2..cf2fc35f6 100644 --- a/roles/upload-logs-s3/tasks/main.yaml +++ b/roles/upload-logs-s3/tasks/main.yaml @@ -36,5 +36,5 @@ zuul_return: data: zuul: - log_url: "{{ upload_results.url }}/" + log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}" when: upload_results is defined diff --git a/roles/upload-logs-swift/README.rst b/roles/upload-logs-swift/README.rst index fb21c7b44..5e2611bc3 100644 --- a/roles/upload-logs-swift/README.rst +++ b/roles/upload-logs-swift/README.rst @@ -76,3 +76,10 @@ This uploads logs to an OpenStack Object Store (Swift) container. More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`. + +.. zuul:rolevar:: zuul_log_storage_proxy_url + + The url of the proxy for the cloud object store. + If you are using zuul-storage-proxy to proxy requests for logs, set this + to the the URL of the log proxy server. When set, this role will replace + the cloud storage endpoint with this value in the log URL returned to Zuul. diff --git a/roles/upload-logs-swift/tasks/main.yaml b/roles/upload-logs-swift/tasks/main.yaml index 2497ec41d..7bb32f98f 100644 --- a/roles/upload-logs-swift/tasks/main.yaml +++ b/roles/upload-logs-swift/tasks/main.yaml @@ -36,7 +36,7 @@ zuul_return: data: zuul: - log_url: "{{ upload_results.url }}/" + log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}" - name: Print upload failures debug: var: upload_results.upload_failures