From 0996433fe5fb8590eda1111e88ba74ab28d4508a Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Tue, 3 Oct 2023 14:56:30 +1100 Subject: [PATCH] s3api: Add basic GET object-lock support Some tooling out there, like Ansible, will always call to see if object-lock is enabled on a bucket/container. This fails as Swift doesn't understand the object-lock or the get object lock api[0]. When you use the get-object-lock-configuration to a bucket in s3 that doesn't have it applied it returns a specific 404: GET /?object-lock HTTP/1.1" 404 None ... ObjectLockConfigurationNotFoundError Object Lock configuration does not exist for this bucket bucket_name 83VQBYP0SENV3VP4 ' This patch doesn't add support for get_object lock, instead it always returns a similar 404 as supplied by s3, so clients know it's not enabled. Also add a object-lock PUT 501 response. [0] https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectLockConfiguration.html Change-Id: Icff8cf57474dfad975a4f45bf2d500c2682c1129 --- .../conf/ceph-known-failures-tempauth.yaml | 4 +- .../middleware/s3api/controllers/__init__.py | 3 + .../s3api/controllers/object_lock.py | 44 ++++++++++++ swift/common/middleware/s3api/s3request.py | 10 ++- swift/common/middleware/s3api/s3response.py | 13 +++- test/functional/s3api/test_bucket.py | 52 ++++++++++++++ test/s3api/test_object_lock.py | 51 ++++++++++++++ .../middleware/s3api/test_object_lock.py | 67 +++++++++++++++++++ 8 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 swift/common/middleware/s3api/controllers/object_lock.py create mode 100644 test/s3api/test_object_lock.py create mode 100644 test/unit/common/middleware/s3api/test_object_lock.py diff --git a/doc/s3api/conf/ceph-known-failures-tempauth.yaml b/doc/s3api/conf/ceph-known-failures-tempauth.yaml index a30bf29c14..0ea8d3daa3 100644 --- a/doc/s3api/conf/ceph-known-failures-tempauth.yaml +++ b/doc/s3api/conf/ceph-known-failures-tempauth.yaml @@ -150,7 +150,6 @@ ceph_s3: s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_retention: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_get_legal_hold_invalid_bucket: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_get_obj_lock: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_object_lock_get_obj_lock_invalid_bucket: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_get_obj_metadata: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention_invalid_bucket: {status: KNOWN} @@ -159,6 +158,9 @@ ceph_s3: s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_bucket: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_days: {status: KNOWN} + s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_status {status: KNOWN} + s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_years {status: KNOWN} + s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_with_days_and_years {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_put_obj_retention: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_put_obj_retention_increase_period: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_put_obj_retention_invalid_bucket: {status: KNOWN} diff --git a/swift/common/middleware/s3api/controllers/__init__.py b/swift/common/middleware/s3api/controllers/__init__.py index ba335b4c67..9e14bd0035 100644 --- a/swift/common/middleware/s3api/controllers/__init__.py +++ b/swift/common/middleware/s3api/controllers/__init__.py @@ -33,6 +33,8 @@ from swift.common.middleware.s3api.controllers.versioning import \ VersioningController from swift.common.middleware.s3api.controllers.tagging import \ TaggingController +from swift.common.middleware.s3api.controllers.object_lock import \ + ObjectLockController __all__ = [ 'Controller', @@ -50,6 +52,7 @@ __all__ = [ 'LoggingStatusController', 'VersioningController', 'TaggingController', + 'ObjectLockController', 'UnsupportedController', ] diff --git a/swift/common/middleware/s3api/controllers/object_lock.py b/swift/common/middleware/s3api/controllers/object_lock.py new file mode 100644 index 0000000000..69a5295727 --- /dev/null +++ b/swift/common/middleware/s3api/controllers/object_lock.py @@ -0,0 +1,44 @@ +# Copyright (c) 2010-2023 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common.utils import public + +from swift.common.middleware.s3api.controllers.base import Controller, \ + bucket_operation, S3NotImplemented +from swift.common.middleware.s3api.s3response import \ + ObjectLockConfigurationNotFoundError + + +class ObjectLockController(Controller): + """ + Handles GET object-lock request, which always returns + Disabled + """ + @public + @bucket_operation + def GET(self, req): + """ + Handles GET object-lock param calls. + """ + raise ObjectLockConfigurationNotFoundError(req.container_name) + + @public + @bucket_operation + def PUT(self, req): + """ + Handles PUT object-lock param calls. + """ + # Basically we don't support it, so return a 501 + raise S3NotImplemented('The requested resource is not implemented') diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index 626377776f..7a6c032d77 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -47,7 +47,7 @@ from swift.common.middleware.s3api.controllers import ServiceController, \ LocationController, LoggingStatusController, PartController, \ UploadController, UploadsController, VersioningController, \ UnsupportedController, S3AclController, BucketController, \ - TaggingController + TaggingController, ObjectLockController from swift.common.middleware.s3api.s3response import AccessDenied, \ InvalidArgument, InvalidDigest, BucketAlreadyOwnedByYou, \ RequestTimeTooSkewed, S3Response, SignatureDoesNotMatch, \ @@ -74,7 +74,8 @@ ALLOWED_SUB_RESOURCES = sorted([ 'versionId', 'versioning', 'versions', 'website', 'response-cache-control', 'response-content-disposition', 'response-content-encoding', 'response-content-language', - 'response-content-type', 'response-expires', 'cors', 'tagging', 'restore' + 'response-content-type', 'response-expires', 'cors', 'tagging', 'restore', + 'object-lock' ]) @@ -103,6 +104,7 @@ def _header_acl_property(resource): """ Set and retrieve the acl in self.headers """ + def getter(self): return getattr(self, '_%s' % resource) @@ -121,6 +123,7 @@ class HashingInput(object): """ wsgi.input wrapper to verify the hash of the input as it's read. """ + def __init__(self, reader, content_length, hasher, expected_hex_hash): self._input = reader self._to_read = content_length @@ -1051,6 +1054,8 @@ class S3Request(swob.Request): return VersioningController if 'tagging' in self.params: return TaggingController + if 'object-lock' in self.params: + return ObjectLockController unsupported = ('notification', 'policy', 'requestPayment', 'torrent', 'website', 'cors', 'restore') @@ -1536,6 +1541,7 @@ class S3AclRequest(S3Request): """ S3Acl request object. """ + def __init__(self, env, app=None, conf=None): super(S3AclRequest, self).__init__(env, app, conf) self.authenticate(app) diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index 25c5900d4f..f74147095a 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -111,6 +111,7 @@ class S3Response(S3ResponseBase, swob.Response): headers instead of Swift's HeaderKeyDict. This also translates Swift specific headers to S3 headers. """ + def __init__(self, *args, **kwargs): swob.Response.__init__(self, *args, **kwargs) @@ -239,7 +240,7 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException): self.info = kwargs.copy() for reserved_key in ('headers', 'body'): if self.info.get(reserved_key): - del(self.info[reserved_key]) + del (self.info[reserved_key]) swob.HTTPException.__init__( self, status=kwargs.pop('status', self._status), @@ -613,6 +614,16 @@ class NoSuchKey(ErrorResponse): ErrorResponse.__init__(self, msg, key=key, *args, **kwargs) +class ObjectLockConfigurationNotFoundError(ErrorResponse): + _status = '404 Not found' + _msg = 'Object Lock configuration does not exist for this bucket' + + def __init__(self, bucket, msg=None, *args, **kwargs): + if not bucket: + raise InternalError() + ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs) + + class NoSuchLifecycleConfiguration(ErrorResponse): _status = '404 Not Found' _msg = 'The lifecycle configuration does not exist. .' diff --git a/test/functional/s3api/test_bucket.py b/test/functional/s3api/test_bucket.py index ed0c411112..b577d4bd0a 100644 --- a/test/functional/s3api/test_bucket.py +++ b/test/functional/s3api/test_bucket.py @@ -567,6 +567,58 @@ class TestS3ApiBucket(S3ApiBaseBoto3): self.assertEqual( ctx.exception.response['Error']['Code'], 'MethodNotAllowed') + def test_bucket_get_object_lock_configuration(self): + bucket = 'bucket' + + # PUT Bucket + resp = self.conn.create_bucket(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + headers = resp['ResponseMetadata']['HTTPHeaders'] + self.assertCommonResponseHeaders(headers) + + # now attempt to get object_lock_configuration from new bucket. + with self.assertRaises(botocore.exceptions.ClientError) as ce: + self.conn.get_object_lock_configuration( + Bucket=bucket) + self.assertEqual( + ce.exception.response['ResponseMetadata']['HTTPStatusCode'], + 404) + self.assertEqual( + ce.exception.response['Error']['Code'], + 'ObjectLockConfigurationNotFoundError') + + self.assertEqual( + str(ce.exception), + 'An error occurred (ObjectLockConfigurationNotFoundError) when ' + 'calling the GetObjectLockConfiguration operation: Object Lock ' + 'configuration does not exist for this bucket') + + def test_bucket_put_object_lock_configuration(self): + bucket = 'bucket' + + # PUT Bucket + resp = self.conn.create_bucket(Bucket=bucket) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + headers = resp['ResponseMetadata']['HTTPHeaders'] + self.assertCommonResponseHeaders(headers) + + # now attempt to get object_lock_configuration from new bucket. + with self.assertRaises(botocore.exceptions.ClientError) as ce: + self.conn.put_object_lock_configuration( + Bucket=bucket, ObjectLockConfiguration={}) + + self.assertEqual( + ce.exception.response['ResponseMetadata']['HTTPStatusCode'], + 501) + self.assertEqual( + ce.exception.response['Error']['Code'], + 'NotImplemented') + + self.assertEqual(str(ce.exception), + 'An error occurred (NotImplemented) when calling ' + 'the PutObjectLockConfiguration operation: The ' + 'requested resource is not implemented') + class TestS3ApiBucketSigV4(TestS3ApiBucket): @classmethod diff --git a/test/s3api/test_object_lock.py b/test/s3api/test_object_lock.py new file mode 100644 index 0000000000..a944008257 --- /dev/null +++ b/test/s3api/test_object_lock.py @@ -0,0 +1,51 @@ +# Copyright (c) 2010-2023 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import botocore + +from test.s3api import BaseS3TestCase + + +class TestObjectLockConfiguration(BaseS3TestCase): + + maxDiff = None + + def setUp(self): + self.client = self.get_s3_client(1) + self.bucket_name = self.create_name('objlock') + resp = self.client.create_bucket(Bucket=self.bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + + def tearDown(self): + self.clear_bucket(self.client, self.bucket_name) + super(TestObjectLockConfiguration, self).tearDown() + + def test_get_object_lock_configuration(self): + with self.assertRaises(botocore.exceptions.ClientError) as ce: + self.client.get_object_lock_configuration( + Bucket=self.bucket_name) + + self.assertEqual( + ce.exception.response['ResponseMetadata']['HTTPStatusCode'], + 404) + self.assertEqual( + ce.exception.response['Error']['Code'], + 'ObjectLockConfigurationNotFoundError') + + self.assertEqual( + str(ce.exception), + 'An error occurred (ObjectLockConfigurationNotFoundError) when ' + 'calling the GetObjectLockConfiguration operation: Object Lock ' + 'configuration does not exist for this bucket') diff --git a/test/unit/common/middleware/s3api/test_object_lock.py b/test/unit/common/middleware/s3api/test_object_lock.py new file mode 100644 index 0000000000..3bdf307e29 --- /dev/null +++ b/test/unit/common/middleware/s3api/test_object_lock.py @@ -0,0 +1,67 @@ +# Copyright (c) 2010-2023 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift.common.swob import Request + +from test.unit.common.middleware.s3api import S3ApiTestCase +from swift.common.middleware.s3api.etree import fromstring + + +class TestS3ApiObjectLock(S3ApiTestCase): + + # The object-lock controller currently only returns a static response + # as disabled. Things like ansible need this. So there isn't much to test. + + def test_get_object_lock(self): + req = Request.blank('/bucket?object-lock', + environ={'REQUEST_METHOD': 'GET', + 'swift.trans_id': 'txt1234'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '404') + elem = fromstring(body, 'Error') + + self.assertTrue(elem.getchildren()) + for child in elem.iterchildren(): + if child.tag == 'Code': + self.assertEqual(child.text, + 'ObjectLockConfigurationNotFoundError') + elif child.tag == 'Message': + self.assertEqual(child.text, + 'Object Lock configuration does not exist ' + 'for this bucket') + elif child.tag == 'RequestId': + self.assertEqual(child.text, + 'txt1234') + elif child.tag == 'BucketName': + self.assertEqual(child.text, 'bucket') + else: + self.fail('Found unknown sub entry') + + def test_put_object_lock(self): + req = Request.blank('/bucket?object-lock', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + # This currently isn't implemented. + self.assertEqual(status.split()[0], '501') + + +if __name__ == '__main__': + unittest.main()