Merge "Update upload-logs roles to support endpoint override"

roles/upload-logs-base/library/zuul_google_storage_upload.py

@@ -93,7 +93,8 @@ class Uploader():
 
         self.dry_run = dry_run
         if dry_run:
-            self.url = 'http://dry-run-url.com/a/path/'
+            self.endpoint = 'http://dry-run-url.com'
+            self.path = '/a/path/'
             return
 
         self.client = client
@@ -107,9 +108,8 @@ class Uploader():
         self.bucket.cors = cors
         self.bucket.website = {"mainPageSuffix": "index.html"}
         self.bucket.update()
-
+        self.endpoint = 'https://storage.googleapis.com'
-        self.url = os.path.join('https://storage.googleapis.com/',
+        self.path = os.path.join(container, self.prefix)
-                                container, self.prefix)
 
     def upload(self, file_list):
         """Spin up thread pool to upload to storage"""
@@ -240,7 +240,7 @@ def run(container, files,
         # Upload.
         uploader = Uploader(client, container, prefix, dry_run)
         uploader.upload(file_list)
-        return uploader.url
+        return uploader.endpoint, uploader.path
 
 
 def ansible_main():
@@ -260,17 +260,18 @@ def ansible_main():
     )
 
     p = module.params
-    url = run(p.get('container'), p.get('files'),
+    endpoint, path = run(p.get('container'), p.get('files'),
-              indexes=p.get('indexes'),
+                         indexes=p.get('indexes'),
-              parent_links=p.get('parent_links'),
+                         parent_links=p.get('parent_links'),
-              topdir_parent_link=p.get('topdir_parent_link'),
+                         topdir_parent_link=p.get('topdir_parent_link'),
-              partition=p.get('partition'),
+                         partition=p.get('partition'),
-              footer=p.get('footer'),
+                         footer=p.get('footer'),
-              prefix=p.get('prefix'),
+                         prefix=p.get('prefix'),
-              credentials_file=p.get('credentials_file'),
+                         credentials_file=p.get('credentials_file'),
-              project=p.get('project'))
+                         project=p.get('project'))
     module.exit_json(changed=True,
-                     url=url)
+                     endpoint=endpoint,
+                     path=path)
 
 
 def cli_main():
@@ -320,17 +321,17 @@ def cli_main():
     if append_footer.lower() == 'none':
         append_footer = None
 
-    url = run(args.container, args.files,
+    _, path = run(args.container, args.files,
-              indexes=not args.no_indexes,
+                  indexes=not args.no_indexes,
-              parent_links=not args.no_parent_links,
+                  parent_links=not args.no_parent_links,
-              topdir_parent_link=args.create_topdir_parent_link,
+                  topdir_parent_link=args.create_topdir_parent_link,
-              partition=args.partition,
+                  partition=args.partition,
-              footer=append_footer,
+                  footer=append_footer,
-              prefix=args.prefix,
+                  prefix=args.prefix,
-              dry_run=args.dry_run,
+                  dry_run=args.dry_run,
-              credentials_file=args.credentials_file,
+                  credentials_file=args.credentials_file,
-              project=args.project)
+                  project=args.project)
-    print(url)
+    print(path)
 
 
 if __name__ == '__main__':

roles/upload-logs-base/library/zuul_s3_upload.py

@@ -63,7 +63,8 @@ class Uploader():
         self.dry_run = dry_run
         self.public = public
         if dry_run:
-            self.url = 'http://dry-run-url.com/a/path/'
+            self.endpoint = 'http://dry-run-url.com'
+            self.path = '/a/path'
             return
 
         self.prefix = prefix or ''
@@ -73,8 +74,7 @@ class Uploader():
         else:
             self.endpoint = 'https://s3.amazonaws.com/'
 
-        self.url = os.path.join(self.endpoint,
+        self.path = os.path.join(bucket, self.prefix)
-                                bucket, self.prefix)
 
         self.s3 = boto3.resource('s3',
                                  endpoint_url=self.endpoint,
@@ -223,7 +223,7 @@ def run(bucket, public, files, endpoint=None,
                             aws_secret_key=aws_secret_key)
         upload_failures = uploader.upload(file_list)
 
-        return uploader.url, upload_failures
+        return uploader.endpoint, uploader.path, upload_failures
 
 
 def ansible_main():
@@ -245,24 +245,28 @@ def ansible_main():
     )
 
     p = module.params
-    url, failures = run(p.get('bucket'),
+    endpoint, path, failures = run(
-                        p.get('public'),
+        p.get('bucket'),
-                        p.get('files'),
+        p.get('public'),
-                        p.get('endpoint'),
+        p.get('files'),
-                        indexes=p.get('indexes'),
+        p.get('endpoint'),
-                        parent_links=p.get('parent_links'),
+        indexes=p.get('indexes'),
-                        topdir_parent_link=p.get('topdir_parent_link'),
+        parent_links=p.get('parent_links'),
-                        partition=p.get('partition'),
+        topdir_parent_link=p.get('topdir_parent_link'),
-                        footer=p.get('footer'),
+        partition=p.get('partition'),
-                        prefix=p.get('prefix'),
+        footer=p.get('footer'),
-                        aws_access_key=p.get('aws_access_key'),
+        prefix=p.get('prefix'),
-                        aws_secret_key=p.get('aws_secret_key'))
+        aws_access_key=p.get('aws_access_key'),
+        aws_secret_key=p.get('aws_secret_key'),
+    )
     if failures:
         module.fail_json(changed=True,
-                         url=url,
+                         endpoint=endpoint,
+                         path=path,
                          failures=failures)
     module.exit_json(changed=True,
-                     url=url,
+                     endpoint=endpoint,
+                     path=path,
                      failures=failures)
 
 
@@ -290,11 +294,10 @@ def cli_main():
         logging.basicConfig(level=logging.DEBUG)
         logging.captureWarnings(True)
 
-    url = run(args.bucket, args.files,
+    _, path, _ = run(args.bucket, not args.no_public, args.files,
-              prefix=args.prefix,
+                     prefix=args.prefix,
-              public=not args.no_public,
+                     endpoint=args.endpoint)
-              endpoint=args.endpoint)
+    print(path)
-    print(url)
 
 
 if __name__ == '__main__':

roles/upload-logs-base/library/zuul_swift_upload.py

@@ -77,7 +77,8 @@ class Uploader():
 
         self.dry_run = dry_run
         if dry_run:
-            self.url = 'http://dry-run-url.com/a/path/'
+            self.endpoint = 'http://dry-run-url.com'
+            self.path = '/a/path'
             return
 
         self.cloud = cloud
@@ -140,8 +141,8 @@ class Uploader():
         else:
             endpoint = self.cloud.object_store.get_endpoint()
             container = os.path.join(endpoint, self.container)
-
+        self.endpoint = endpoint
-        self.url = os.path.join(container, self.prefix)
+        self.path = os.path.join(self.container, self.prefix)
 
     def upload(self, file_list):
         """Spin up thread pool to upload to swift"""
@@ -288,7 +289,7 @@ def run(cloud, container, files,
         uploader = Uploader(cloud, container, prefix, delete_after,
                             public, dry_run)
         upload_failures = uploader.upload(file_list)
-        return uploader.url, upload_failures
+        return uploader.endpoint, uploader.path, upload_failures
 
 
 def ansible_main():
@@ -311,7 +312,7 @@ def ansible_main():
     p = module.params
     cloud = get_cloud(p.get('cloud'))
     try:
-        url, upload_failures = run(
+        endpoint, path, upload_failures = run(
             cloud, p.get('container'), p.get('files'),
             indexes=p.get('indexes'),
             parent_links=p.get('parent_links'),
@@ -334,7 +335,8 @@ def ansible_main():
             region_name=cloud.config.region_name)
     module.exit_json(
         changed=True,
-        url=url,
+        endpoint=endpoint,
+        path=path,
         upload_failures=upload_failures,
     )
 
@@ -394,7 +396,7 @@ def cli_main():
     if append_footer.lower() == 'none':
         append_footer = None
 
-    url, _ = run(
+    _, path, _ = run(
         get_cloud(args.cloud), args.container, args.files,
         indexes=not args.no_indexes,
         parent_links=not args.no_parent_links,
@@ -406,7 +408,7 @@ def cli_main():
         public=not args.no_public,
         dry_run=args.dry_run
     )
-    print(url)
+    print(path)
 
 
 if __name__ == '__main__':

roles/upload-logs-gcs/README.rst

@@ -78,3 +78,10 @@ Google Cloud Application Default Credentials.
    :zuul:rolevar:`upload-logs-gcs.zuul_log_credentials_file`, the name
    of the Google Cloud project of the log container must also be
    supplied.
+
+.. zuul:rolevar:: zuul_log_storage_proxy_url
+
+   The url of the proxy for the cloud object store.
+   If you are using zuul-storage-proxy to proxy requests for logs, set this
+   to the the URL of the log proxy server.  When set, this role will replace
+   the cloud storage endpoint with this value in the log URL returned to Zuul.

roles/upload-logs-gcs/tasks/main.yaml

@@ -34,5 +34,5 @@
   zuul_return:
     data:
       zuul:
-        log_url: "{{ upload_results.url }}/"
+        log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
   when: upload_results is defined

roles/upload-logs-s3/README.rst

@@ -76,3 +76,10 @@ installed in the Ansible environment on the Zuul executor.
 
    The endpoint to use when uploading logs to an s3 compatible service.
    By default this will be automatically constructed by boto but should be set when working with non-aws hosted s3 service.
+
+.. zuul:rolevar:: zuul_log_storage_proxy_url
+
+   The url of the proxy for the cloud object store.
+   If you are using zuul-storage-proxy to proxy requests for logs, set this
+   to the the URL of the log proxy server.  When set, this role will replace
+   the cloud storage endpoint with this value in the log URL returned to Zuul.

roles/upload-logs-s3/tasks/main.yaml

@@ -36,5 +36,5 @@
   zuul_return:
     data:
       zuul:
-        log_url: "{{ upload_results.url }}/"
+        log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
   when: upload_results is defined

roles/upload-logs-swift/README.rst

@@ -76,3 +76,10 @@ This uploads logs to an OpenStack Object Store (Swift) container.
 
    More details can be found at
    :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
+
+.. zuul:rolevar:: zuul_log_storage_proxy_url
+
+   The url of the proxy for the cloud object store.
+   If you are using zuul-storage-proxy to proxy requests for logs, set this
+   to the the URL of the log proxy server.  When set, this role will replace
+   the cloud storage endpoint with this value in the log URL returned to Zuul.

roles/upload-logs-swift/tasks/main.yaml

@@ -36,7 +36,7 @@
       zuul_return:
         data:
           zuul:
-            log_url: "{{ upload_results.url }}/"
+            log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
     - name: Print upload failures
       debug:
         var: upload_results.upload_failures