zuul/zuul-jobs
Code Issues Proposed changes

Merge "Raise connection pool for boto3 in s3 upload role"

Browse Source
This commit is contained in:
Zuul
2025-09-18 17:19:36 +00:00
committed by Gerrit Code Review
parent f50d0b2248 43ca512ac5
commit 161364dee4
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
roles/upload-logs-base/library/zuul_s3_upload.py
View File

@@ -38,6 +38,7 @@ import sys
import threading import threading
import boto3 import boto3
import botocore
from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import AnsibleModule
try: try:
@@ -113,7 +114,13 @@ class Uploader():
self.url = os.path.join(return_endpoint, self.url = os.path.join(return_endpoint,
self.prefix) self.prefix)
# The default for max_pool_connections is 10, which is easily
# exhausted if the upload threads overwhelm them.
client_config = botocore.config.Config(
max_pool_connections=MAX_UPLOAD_THREADS * 4)
self.s3 = boto3.resource('s3', self.s3 = boto3.resource('s3',
config=client_config,
endpoint_url=self.endpoint, endpoint_url=self.endpoint,
**credential_args) **credential_args)
self.bucket = self.s3.Bucket(bucket) self.bucket = self.s3.Bucket(bucket)