Create segment container w/ same policy as primary
When users upload an MPU object, s3api will automatically create a segment container if one doesn't already exist. Currently, s3api will create the segment bucket using the cluster's default storage policy. This patch changes that behavior to use the same storage policy as the primary bucket. Change-Id: Ib64a06868bd3670a1d4a1860ac29122e1ede7c39 Closes-Bug: 1832390
This commit is contained in:
Thiago da Silva
parent
1c0661e920
commit
d667affb6f
@ -60,6 +60,7 @@ Static Large Object when the multipart upload is completed.
|
||||
"""
|
||||
|
||||
import binascii
|
||||
import copy
|
||||
from hashlib import md5
|
||||
import os
|
||||
import re
|
||||
@ -86,6 +87,7 @@ from swift.common.middleware.s3api.utils import unique_id, \
|
||||
MULTIUPLOAD_SUFFIX, S3Timestamp, sysmeta_header
|
||||
from swift.common.middleware.s3api.etree import Element, SubElement, \
|
||||
fromstring, tostring, XMLSyntaxError, DocumentInvalid
|
||||
from swift.common.storage_policy import POLICIES
|
||||
|
||||
DEFAULT_MAX_PARTS_LISTING = 1000
|
||||
DEFAULT_MAX_UPLOADS = 1000
|
||||
@ -364,8 +366,7 @@ class UploadsController(Controller):
|
||||
# Create a unique S3 upload id from UUID to avoid duplicates.
|
||||
upload_id = unique_id()
|
||||
|
||||
orig_container = req.container_name
|
||||
seg_container = orig_container + MULTIUPLOAD_SUFFIX
|
||||
seg_container = req.container_name + MULTIUPLOAD_SUFFIX
|
||||
content_type = req.headers.get('Content-Type')
|
||||
if content_type:
|
||||
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
|
||||
@ -376,15 +377,21 @@ class UploadsController(Controller):
|
||||
req.headers['Content-Type'] = 'application/directory'
|
||||
|
||||
try:
|
||||
req.container_name = seg_container
|
||||
req.get_container_info(self.app)
|
||||
seg_req = copy.copy(req)
|
||||
seg_req.environ = copy.copy(req.environ)
|
||||
seg_req.container_name = seg_container
|
||||
seg_req.get_container_info(self.app)
|
||||
except NoSuchBucket:
|
||||
try:
|
||||
req.get_response(self.app, 'PUT', seg_container, '')
|
||||
# multi-upload bucket doesn't exist, create one with
|
||||
# same storage policy as the primary bucket
|
||||
info = req.get_container_info(self.app)
|
||||
policy_name = POLICIES[info['storage_policy']].name
|
||||
hdrs = {'X-Storage-Policy': policy_name}
|
||||
seg_req.get_response(self.app, 'PUT', seg_container, '',
|
||||
headers=hdrs)
|
||||
except (BucketAlreadyExists, BucketAlreadyOwnedByYou):
|
||||
pass
|
||||
finally:
|
||||
req.container_name = orig_container
|
||||
|
||||
obj = '%s/%s' % (req.object_name, upload_id)
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ from swift.common import swob
|
||||
from swift.common.swob import Request
|
||||
from swift.common.utils import json
|
||||
|
||||
from test.unit import FakeMemcache
|
||||
from test.unit import FakeMemcache, patch_policies
|
||||
from test.unit.common.middleware.s3api import S3ApiTestCase
|
||||
from test.unit.common.middleware.s3api.helpers import UnreadableInput
|
||||
from swift.common.middleware.s3api.etree import fromstring, tostring
|
||||
@ -36,6 +36,7 @@ from test.unit.common.middleware.s3api.test_s3_acl import s3acl
|
||||
from swift.common.middleware.s3api.utils import sysmeta_header, mktime, \
|
||||
S3Timestamp
|
||||
from swift.common.middleware.s3api.s3request import MAX_32BIT_INT
|
||||
from swift.common.storage_policy import StoragePolicy
|
||||
from swift.proxy.controllers.base import get_cache_key
|
||||
|
||||
XML = '<CompleteMultipartUpload>' \
|
||||
@ -555,7 +556,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||
@patch('swift.common.middleware.s3api.controllers.'
|
||||
'multi_upload.unique_id', lambda: 'X')
|
||||
def _test_object_multipart_upload_initiate(self, headers, cache=None,
|
||||
bucket_exists=True):
|
||||
bucket_exists=True,
|
||||
expected_policy=None):
|
||||
headers.update({
|
||||
'Authorization': 'AWS test:tester:hmac',
|
||||
'Date': self.get_date_header(),
|
||||
@ -584,6 +586,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||
('PUT', '/v1/AUTH_test/bucket+segments'),
|
||||
('PUT', '/v1/AUTH_test/bucket+segments/object/X'),
|
||||
], self.swift.calls)
|
||||
if expected_policy:
|
||||
_, _, req_headers = self.swift.calls_with_headers[-2]
|
||||
self.assertEqual(req_headers.get('X-Storage-Policy'),
|
||||
expected_policy)
|
||||
self.swift.clear_calls()
|
||||
|
||||
def test_object_multipart_upload_initiate_with_segment_bucket(self):
|
||||
@ -601,6 +607,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||
self.swift.register('PUT', '/v1/AUTH_test/bucket+segments',
|
||||
swob.HTTPCreated, {}, None)
|
||||
fake_memcache = FakeMemcache()
|
||||
fake_memcache.store[get_cache_key(
|
||||
'AUTH_test', 'bucket')] = {'status': 204}
|
||||
fake_memcache.store[get_cache_key(
|
||||
'AUTH_test', 'bucket+segments')] = {'status': 404}
|
||||
self._test_object_multipart_upload_initiate({}, fake_memcache,
|
||||
@ -613,6 +621,33 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||
fake_memcache,
|
||||
bucket_exists=False)
|
||||
|
||||
@patch_policies([
|
||||
StoragePolicy(0, 'gold', is_default=True),
|
||||
StoragePolicy(1, 'silver')])
|
||||
def test_object_mpu_initiate_without_segment_bucket_same_container(self):
|
||||
self.swift.register('PUT', '/v1/AUTH_test/bucket+segments',
|
||||
swob.HTTPCreated,
|
||||
{'X-Storage-Policy': 'silver'}, None)
|
||||
fake_memcache = FakeMemcache()
|
||||
fake_memcache.store[get_cache_key(
|
||||
'AUTH_test', 'bucket')] = {'status': 204,
|
||||
'storage_policy': '1'}
|
||||
fake_memcache.store[get_cache_key(
|
||||
'AUTH_test', 'bucket+segments')] = {'status': 404}
|
||||
self.s3api.conf.derived_container_policy_use_default = False
|
||||
self._test_object_multipart_upload_initiate({}, fake_memcache,
|
||||
bucket_exists=False,
|
||||
expected_policy='silver')
|
||||
self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'},
|
||||
fake_memcache,
|
||||
bucket_exists=False,
|
||||
expected_policy='silver')
|
||||
self._test_object_multipart_upload_initiate(
|
||||
{'Content-MD5': base64.b64encode(b'blahblahblahblah').strip()},
|
||||
fake_memcache,
|
||||
bucket_exists=False,
|
||||
expected_policy='silver')
|
||||
|
||||
@patch('swift.common.middleware.s3api.controllers.multi_upload.'
|
||||
'unique_id', lambda: 'X')
|
||||
def _test_object_multipart_upload_initiate_s3acl(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user