openstack/swift
Code Issues Proposed changes

Merge "Move the mount checking into DiskFile constructor"

Browse Source
This commit is contained in:
Jenkins 2013-07-23 23:01:44 +00:00 committed by Gerrit Code Review
commit e777bca988
5 changed files with 55 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
swift/common/exceptions.py
View File

@ -38,15 +38,19 @@ class DiskFileError(SwiftException):
pass
class DiskFileCollision(SwiftException):
class DiskFileCollision(DiskFileError):
pass
class DiskFileNotExist(SwiftException):
class DiskFileNotExist(DiskFileError):
pass
class DiskFileNoSpace(SwiftException):
class DiskFileNoSpace(DiskFileError):
pass
class DiskFileDeviceUnavailable(DiskFileError):
pass

4
swift/obj/auditor.py
View File

@ -170,10 +170,10 @@ class AuditorWorker(object):
try:
try:
obj_size = df.get_data_file_size()
except DiskFileError as e:
raise AuditException(str(e))
except DiskFileNotExist:
return
except DiskFileError as e:
raise AuditException(str(e))
if self.stats_sizes:
self.record_stats(obj_size)
if self.zero_byte_only_at_fps and obj_size:

13
swift/obj/diskfile.py
View File

@ -28,11 +28,12 @@ from contextlib import contextmanager
from xattr import getxattr, setxattr
from eventlet import Timeout
from swift.common.constraints import check_mount
from swift.common.utils import mkdirs, normalize_timestamp, \
storage_directory, hash_path, renamer, fallocate, fsync, \
fdatasync, drop_buffer_cache, ThreadPool
from swift.common.exceptions import DiskFileError, \
DiskFileNotExist, DiskFileCollision, DiskFileNoSpace
from swift.common.exceptions import DiskFileError, DiskFileNotExist, \
DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable
from swift.obj.base import invalidate_hash, \
quarantine_renamer
from swift.common.swob import multi_range_iterator
@ -170,8 +171,10 @@ class DiskFile(object):
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536,
bytes_per_sync=(512 * 1024 * 1024), iter_hook=None,
threadpool=None, obj_dir='objects',
threadpool=None, obj_dir='objects', mount_check=False,
disallowed_metadata_keys=None):
if mount_check and not check_mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
@ -297,11 +300,11 @@ class DiskFile(object):
"""Check if file needs to be quarantined"""
try:
self.get_data_file_size()
except DiskFileNotExist:
return
except DiskFileError:
self.quarantine()
return
except DiskFileNotExist:
return
if self.iter_etag and self.started_at_0 and self.read_to_eof and \
'ETag' in self.metadata and \

38
swift/obj/server.py
View File

@ -35,7 +35,8 @@ from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, check_mount, \
check_float, check_utf8
from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
DiskFileNotExist, DiskFileCollision, DiskFileNoSpace
DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, \
DiskFileDeviceUnavailable
from swift.obj.base import get_hashes
from swift.common.http import is_success
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
@ -103,6 +104,7 @@ class ObjectController(object):
def _diskfile(self, device, partition, account, container, obj, **kwargs):
"""Utility method for instantiating a DiskFile."""
kwargs.setdefault('mount_check', self.mount_check)
kwargs.setdefault('bytes_per_sync', self.bytes_per_sync)
kwargs.setdefault('disk_chunk_size', self.disk_chunk_size)
kwargs.setdefault('logger', self.logger)
@ -293,9 +295,11 @@ class ObjectController(object):
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file = self._diskfile(device, partition, account, container, obj)
if disk_file.is_deleted() or disk_file.is_expired():
return HTTPNotFound(request=request)
try:
@ -348,9 +352,11 @@ class ObjectController(object):
except ValueError as e:
return HTTPBadRequest(body=str(e), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file = self._diskfile(device, partition, account, container, obj)
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
orig_timestamp = disk_file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
@ -429,10 +435,11 @@ class ObjectController(object):
except ValueError, err:
return HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
try:
disk_file = self._diskfile(device, partition, account, container,
obj, keep_data_fp=True, iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file = self._diskfile(device, partition, account, container, obj,
keep_data_fp=True, iter_hook=sleep)
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
@ -509,9 +516,11 @@ class ObjectController(object):
resp.content_type = 'text/plain'
resp.body = str(err)
return resp
if self.mount_check and not check_mount(self.devices, device):
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file = self._diskfile(device, partition, account, container, obj)
if disk_file.is_deleted() or disk_file.is_expired():
return HTTPNotFound(request=request)
try:
@ -550,10 +559,11 @@ class ObjectController(object):
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
response_class = HTTPNoContent
disk_file = self._diskfile(device, partition, account, container, obj)
if 'x-if-delete-at' in request.headers and \
int(request.headers['x-if-delete-at']) != \
int(disk_file.metadata.get('X-Delete-At') or 0):
@ -563,6 +573,8 @@ class ObjectController(object):
orig_timestamp = disk_file.metadata.get('X-Timestamp')
if disk_file.is_deleted() or disk_file.is_expired():
response_class = HTTPNotFound
else:
response_class = HTTPNoContent
metadata = {
'X-Timestamp': request.headers['X-Timestamp'], 'deleted': True,
}

19
test/unit/obj/test_diskfile.py
View File

@ -37,7 +37,7 @@ from swift.common import utils
from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
NullLogger, storage_directory, public, \
replication
from swift.common.exceptions import DiskFileNotExist
from swift.common.exceptions import DiskFileNotExist, DiskFileDeviceUnavailable
from swift.common import constraints
from eventlet import tpool
from swift.common.swob import Request, HeaderKeyDict
@ -56,7 +56,7 @@ class TestDiskFile(unittest.TestCase):
def tearDown(self):
""" Tear down for testing swift.obj.diskfile"""
rmtree(os.path.dirname(self.testdir))
tpool.execute = self._orig_tpool_exc
tpool.execute = self._orig_tpool_exc
def _create_test_file(self, data, keep_data_fp=True):
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',
@ -207,7 +207,7 @@ class TestDiskFile(unittest.TestCase):
def _get_disk_file(self, invalid_type=None, obj_name='o',
fsize=1024, csize=8, mark_deleted=False, ts=None,
iter_hook=None):
iter_hook=None, mount_check=False):
'''returns a DiskFile'''
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',
obj_name, FakeLogger())
@ -245,9 +245,9 @@ class TestDiskFile(unittest.TestCase):
df.put_metadata(metadata, tombstone=True)
df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',
obj_name, FakeLogger(),
keep_data_fp=True, disk_chunk_size=csize,
iter_hook=iter_hook)
obj_name, FakeLogger(),
keep_data_fp=True, disk_chunk_size=csize,
iter_hook=iter_hook, mount_check=mount_check)
if invalid_type == 'Zero-Byte':
os.remove(df.data_file)
fp = open(df.data_file, 'w')
@ -380,3 +380,10 @@ class TestDiskFile(unittest.TestCase):
self.assertFalse(os.path.isfile(df.data_file))
self.assert_(os.path.isdir(quar_dir))
self.assertEquals(df.quarantine(), None)
def test_mount_checking(self):
def _mock_ismount(*args, **kwargs):
return False
with mock.patch("os.path.ismount", _mock_ismount):
self.assertRaises(DiskFileDeviceUnavailable, self._get_disk_file,
mount_check=True)