openstack/swift
Code Issues Proposed changes

Partially revert "Pull libc-related functions out to a separate module"

Browse Source 
This reverts the fallocate- and punch_hole-related parts of commit
c78a5962b5.

Closes-Bug: #2031035
Related-Change: I3e26f8d4e5de0835212ebc2314cac713950c85d7
Change-Id: I8050296d6982f70bb64a63765b25d287a144cb8d
This commit is contained in:
Tim Burke 2023-08-17 09:49:50 -07:00 committed by Tim Burke
parent 9e065e2d23
commit 1edf7df755
5 changed files with 581 additions and 580 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
swift/common/utils/__init__.py
View File

@ -39,6 +39,8 @@ import functools
import email.parser import email.parser
from random import random, shuffle from random import random, shuffle
from contextlib import contextmanager, closing from contextlib import contextmanager, closing
import ctypes
import ctypes.util
from optparse import OptionParser from optparse import OptionParser
import traceback import traceback
import warnings import warnings
@ -94,13 +96,10 @@ from swift.common.registry import register_swift_info, get_swift_info # noqa
from swift.common.utils.libc import ( # noqa from swift.common.utils.libc import ( # noqa
F_SETPIPE_SZ, F_SETPIPE_SZ,
load_libc_function, load_libc_function,
config_fallocate_value,
disable_fallocate,
fallocate,
punch_hole,
drop_buffer_cache, drop_buffer_cache,
get_md5_socket, get_md5_socket,
modify_priority, modify_priority,
_LibcWrapper,
) )
from swift.common.utils.timestamp import ( # noqa from swift.common.utils.timestamp import ( # noqa
NORMAL_FORMAT, NORMAL_FORMAT,
@ -129,6 +128,21 @@ import logging
NOTICE = 25 NOTICE = 25
# These are lazily pulled from libc elsewhere
_sys_fallocate = None
# If set to non-zero, fallocate routines will fail based on free space
# available being at or below this amount, in bytes.
FALLOCATE_RESERVE = 0
# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
# the number of bytes (False).
FALLOCATE_IS_PERCENT = False
# from /usr/include/linux/falloc.h
FALLOC_FL_KEEP_SIZE = 1
FALLOC_FL_PUNCH_HOLE = 2
# Used by hash_path to offer a bit more security when generating hashes for # Used by hash_path to offer a bit more security when generating hashes for
# paths. It simply appends this value to all paths; guessing the hash a path # paths. It simply appends this value to all paths; guessing the hash a path
# will end up with would also require knowing this suffix. # will end up with would also require knowing this suffix.
@ -137,6 +151,10 @@ HASH_PATH_PREFIX = b''
SWIFT_CONF_FILE = '/etc/swift/swift.conf' SWIFT_CONF_FILE = '/etc/swift/swift.conf'
# These constants are Linux-specific, and Python doesn't seem to know
# about them. We ask anyway just in case that ever gets fixed.
#
# The values were copied from the Linux 3.x kernel headers.
O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY) O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY)
MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e' MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e'
@ -672,6 +690,25 @@ def get_trans_id_time(trans_id):
return None return None
def config_fallocate_value(reserve_value):
"""
Returns fallocate reserve_value as an int or float.
Returns is_percent as a boolean.
Returns a ValueError on invalid fallocate value.
"""
try:
if str(reserve_value[-1:]) == '%':
reserve_value = float(reserve_value[:-1])
is_percent = True
else:
reserve_value = int(reserve_value)
is_percent = False
except ValueError:
raise ValueError('Error: %s is an invalid value for fallocate'
'_reserve.' % reserve_value)
return reserve_value, is_percent
class FileLikeIter(object): class FileLikeIter(object):
def __init__(self, iterable): def __init__(self, iterable):
@ -822,6 +859,116 @@ def fs_has_free_space(fs_path, space_needed, is_percent):
return free_bytes >= space_needed return free_bytes >= space_needed
_fallocate_enabled = True
_fallocate_warned_about_missing = False
_sys_fallocate = _LibcWrapper('fallocate')
_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
def disable_fallocate():
global _fallocate_enabled
_fallocate_enabled = False
def fallocate(fd, size, offset=0):
"""
Pre-allocate disk space for a file.
This function can be disabled by calling disable_fallocate(). If no
suitable C function is available in libc, this function is a no-op.
:param fd: file descriptor
:param size: size to allocate (in bytes)
"""
global _fallocate_enabled
if not _fallocate_enabled:
return
if size < 0:
size = 0 # Done historically; not really sure why
if size >= (1 << 63):
raise ValueError('size must be less than 2 ** 63')
if offset < 0:
raise ValueError('offset must be non-negative')
if offset >= (1 << 63):
raise ValueError('offset must be less than 2 ** 63')
# Make sure there's some (configurable) amount of free space in
# addition to the number of bytes we're allocating.
if FALLOCATE_RESERVE:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - size
if FALLOCATE_IS_PERCENT:
free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
if float(free) <= float(FALLOCATE_RESERVE):
raise OSError(
errno.ENOSPC,
'FALLOCATE_RESERVE fail %g <= %g' %
(free, FALLOCATE_RESERVE))
if _sys_fallocate.available:
# Parameters are (fd, mode, offset, length).
#
# mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
# affecting the reported file size).
ret = _sys_fallocate(
fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
ctypes.c_uint64(size))
err = ctypes.get_errno()
elif _sys_posix_fallocate.available:
# Parameters are (fd, offset, length).
ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
ctypes.c_uint64(size))
err = ctypes.get_errno()
else:
# No suitable fallocate-like function is in our libc. Warn about it,
# but just once per process, and then do nothing.
global _fallocate_warned_about_missing
if not _fallocate_warned_about_missing:
logging.warning("Unable to locate fallocate, posix_fallocate in "
"libc. Leaving as a no-op.")
_fallocate_warned_about_missing = True
return
if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
errno.EINVAL):
raise OSError(err, 'Unable to fallocate(%s)' % size)
def punch_hole(fd, offset, length):
"""
De-allocate disk space in the middle of a file.
:param fd: file descriptor
:param offset: index of first byte to de-allocate
:param length: number of bytes to de-allocate
"""
if offset < 0:
raise ValueError('offset must be non-negative')
if offset >= (1 << 63):
raise ValueError('offset must be less than 2 ** 63')
if length <= 0:
raise ValueError('length must be positive')
if length >= (1 << 63):
raise ValueError('length must be less than 2 ** 63')
if _sys_fallocate.available:
# Parameters are (fd, mode, offset, length).
ret = _sys_fallocate(
fd,
FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
ctypes.c_uint64(offset),
ctypes.c_uint64(length))
err = ctypes.get_errno()
if ret and err:
mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
fd, mode_str, offset, length))
else:
raise OSError(errno.ENOTSUP,
'No suitable C function found for hole punching')
def fsync(fd): def fsync(fd):
""" """
Sync modified file data and metadata to disk. Sync modified file data and metadata to disk.

142
swift/common/utils/libc.py
View File

@ -17,7 +17,6 @@
import ctypes import ctypes
import ctypes.util import ctypes.util
import errno
import fcntl import fcntl
import logging import logging
import os import os
@ -26,7 +25,6 @@ import socket
# These are lazily pulled from libc elsewhere # These are lazily pulled from libc elsewhere
_sys_fallocate = None
_posix_fadvise = None _posix_fadvise = None
_libc_socket = None _libc_socket = None
_libc_bind = None _libc_bind = None
@ -36,17 +34,6 @@ _libc_setpriority = None
# see man -s 2 syscall # see man -s 2 syscall
_posix_syscall = None _posix_syscall = None
# If set to non-zero, fallocate routines will fail based on free space
# available being at or below this amount, in bytes.
FALLOCATE_RESERVE = 0
# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or
# the number of bytes (False).
FALLOCATE_IS_PERCENT = False
# from /usr/include/linux/falloc.h
FALLOC_FL_KEEP_SIZE = 1
FALLOC_FL_PUNCH_HOLE = 2
# from /usr/src/linux-headers-*/include/uapi/linux/resource.h # from /usr/src/linux-headers-*/include/uapi/linux/resource.h
PRIO_PROCESS = 0 PRIO_PROCESS = 0
@ -191,135 +178,6 @@ class _LibcWrapper(object):
"No function %r found in libc" % self._func_name) "No function %r found in libc" % self._func_name)
def config_fallocate_value(reserve_value):
"""
Returns fallocate reserve_value as an int or float.
Returns is_percent as a boolean.
Returns a ValueError on invalid fallocate value.
"""
try:
if str(reserve_value[-1:]) == '%':
reserve_value = float(reserve_value[:-1])
is_percent = True
else:
reserve_value = int(reserve_value)
is_percent = False
except ValueError:
raise ValueError('Error: %s is an invalid value for fallocate'
'_reserve.' % reserve_value)
return reserve_value, is_percent
_fallocate_enabled = True
_fallocate_warned_about_missing = False
_sys_fallocate = _LibcWrapper('fallocate')
_sys_posix_fallocate = _LibcWrapper('posix_fallocate')
def disable_fallocate():
global _fallocate_enabled
_fallocate_enabled = False
def fallocate(fd, size, offset=0):
"""
Pre-allocate disk space for a file.
This function can be disabled by calling disable_fallocate(). If no
suitable C function is available in libc, this function is a no-op.
:param fd: file descriptor
:param size: size to allocate (in bytes)
"""
global _fallocate_enabled
if not _fallocate_enabled:
return
if size < 0:
size = 0 # Done historically; not really sure why
if size >= (1 << 63):
raise ValueError('size must be less than 2 ** 63')
if offset < 0:
raise ValueError('offset must be non-negative')
if offset >= (1 << 63):
raise ValueError('offset must be less than 2 ** 63')
# Make sure there's some (configurable) amount of free space in
# addition to the number of bytes we're allocating.
if FALLOCATE_RESERVE:
st = os.fstatvfs(fd)
free = st.f_frsize * st.f_bavail - size
if FALLOCATE_IS_PERCENT:
free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100
if float(free) <= float(FALLOCATE_RESERVE):
raise OSError(
errno.ENOSPC,
'FALLOCATE_RESERVE fail %g <= %g' %
(free, FALLOCATE_RESERVE))
if _sys_fallocate.available:
# Parameters are (fd, mode, offset, length).
#
# mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without
# affecting the reported file size).
ret = _sys_fallocate(
fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset),
ctypes.c_uint64(size))
err = ctypes.get_errno()
elif _sys_posix_fallocate.available:
# Parameters are (fd, offset, length).
ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset),
ctypes.c_uint64(size))
err = ctypes.get_errno()
else:
# No suitable fallocate-like function is in our libc. Warn about it,
# but just once per process, and then do nothing.
global _fallocate_warned_about_missing
if not _fallocate_warned_about_missing:
logging.warning("Unable to locate fallocate, posix_fallocate in "
"libc. Leaving as a no-op.")
_fallocate_warned_about_missing = True
return
if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP,
errno.EINVAL):
raise OSError(err, 'Unable to fallocate(%s)' % size)
def punch_hole(fd, offset, length):
"""
De-allocate disk space in the middle of a file.
:param fd: file descriptor
:param offset: index of first byte to de-allocate
:param length: number of bytes to de-allocate
"""
if offset < 0:
raise ValueError('offset must be non-negative')
if offset >= (1 << 63):
raise ValueError('offset must be less than 2 ** 63')
if length <= 0:
raise ValueError('length must be positive')
if length >= (1 << 63):
raise ValueError('length must be less than 2 ** 63')
if _sys_fallocate.available:
# Parameters are (fd, mode, offset, length).
ret = _sys_fallocate(
fd,
FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
ctypes.c_uint64(offset),
ctypes.c_uint64(length))
err = ctypes.get_errno()
if ret and err:
mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE"
raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % (
fd, mode_str, offset, length))
else:
raise OSError(errno.ENOTSUP,
'No suitable C function found for hole punching')
def drop_buffer_cache(fd, offset, length): def drop_buffer_cache(fd, offset, length):
""" """
Drop 'buffer' cache for the given range of the given file. Drop 'buffer' cache for the given range of the given file.

429
test/unit/common/test_utils.py
View File

@ -2712,6 +2712,44 @@ cluster_dfw1 = http://dfw1.host/v1/
ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright') ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright')
self.assertIsNone(ts) self.assertIsNone(ts)
def test_config_fallocate_value(self):
fallocate_value, is_percent = utils.config_fallocate_value('10%')
self.assertEqual(fallocate_value, 10)
self.assertTrue(is_percent)
fallocate_value, is_percent = utils.config_fallocate_value('10')
self.assertEqual(fallocate_value, 10)
self.assertFalse(is_percent)
try:
fallocate_value, is_percent = utils.config_fallocate_value('ab%')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = utils.config_fallocate_value('ab')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: ab is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = utils.config_fallocate_value('1%%')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = utils.config_fallocate_value('10.0')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
'fallocate_reserve.')
fallocate_value, is_percent = utils.config_fallocate_value('10.5%')
self.assertEqual(fallocate_value, 10.5)
self.assertTrue(is_percent)
fallocate_value, is_percent = utils.config_fallocate_value('10.000%')
self.assertEqual(fallocate_value, 10.000)
self.assertTrue(is_percent)
def test_lock_file(self): def test_lock_file(self):
flags = os.O_CREAT | os.O_RDWR flags = os.O_CREAT | os.O_RDWR
with NamedTemporaryFile(delete=False) as nt: with NamedTemporaryFile(delete=False) as nt:
@ -8785,6 +8823,397 @@ class TestShardRangeList(unittest.TestCase):
do_test([utils.ShardRange.ACTIVE])) do_test([utils.ShardRange.ACTIVE]))
@patch('ctypes.get_errno')
@patch.object(utils, '_sys_posix_fallocate')
@patch.object(utils, '_sys_fallocate')
@patch.object(utils, 'FALLOCATE_RESERVE', 0)
class TestFallocate(unittest.TestCase):
def test_fallocate(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
utils.fallocate(1234, 5000 * 2 ** 20)
# We can't use sys_fallocate_mock.assert_called_once_with because no
# two ctypes.c_uint64 objects are equal even if their values are
# equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 0)
self.assertEqual(args[3].value, 5000 * 2 ** 20)
sys_posix_fallocate_mock.assert_not_called()
def test_fallocate_offset(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 3 * 2 ** 30)
self.assertEqual(args[3].value, 5000 * 2 ** 20)
sys_posix_fallocate_mock.assert_not_called()
def test_fallocate_fatal_error(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
get_errno_mock.return_value = errno.EIO
with self.assertRaises(OSError) as cm:
utils.fallocate(1234, 5000 * 2 ** 20)
self.assertEqual(cm.exception.errno, errno.EIO)
def test_fallocate_silent_errors(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
get_errno_mock.return_value = silent_error
try:
utils.fallocate(1234, 5678)
except OSError:
self.fail("fallocate() raised an error on %d", silent_error)
def test_posix_fallocate_fallback(self, sys_fallocate_mock,
sys_posix_fallocate_mock,
get_errno_mock):
sys_fallocate_mock.available = False
sys_fallocate_mock.side_effect = NotImplementedError
sys_posix_fallocate_mock.available = True
sys_posix_fallocate_mock.return_value = 0
utils.fallocate(1234, 567890)
sys_fallocate_mock.assert_not_called()
calls = sys_posix_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 3)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1].value, 0)
self.assertEqual(args[2].value, 567890)
def test_posix_fallocate_offset(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
sys_fallocate_mock.side_effect = NotImplementedError
sys_posix_fallocate_mock.available = True
sys_posix_fallocate_mock.return_value = 0
utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
calls = sys_posix_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 3)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1].value, 3 * 2 ** 30)
self.assertEqual(args[2].value, 5000 * 2 ** 20)
sys_fallocate_mock.assert_not_called()
def test_no_fallocates_available(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
sys_posix_fallocate_mock.available = False
with mock.patch("logging.warning") as warning_mock, \
mock.patch.object(utils, "_fallocate_warned_about_missing",
False):
utils.fallocate(321, 654)
utils.fallocate(321, 654)
sys_fallocate_mock.assert_not_called()
sys_posix_fallocate_mock.assert_not_called()
get_errno_mock.assert_not_called()
self.assertEqual(len(warning_mock.mock_calls), 1)
def test_arg_bounds(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
with self.assertRaises(ValueError):
utils.fallocate(0, 1 << 64, 0)
with self.assertRaises(ValueError):
utils.fallocate(0, 0, -1)
with self.assertRaises(ValueError):
utils.fallocate(0, 0, 1 << 64)
self.assertEqual([], sys_fallocate_mock.mock_calls)
# sanity check
utils.fallocate(0, 0, 0)
self.assertEqual(
[mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
# Go confirm the ctypes values separately; apparently == doesn't
# work the way you'd expect with ctypes :-/
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
sys_fallocate_mock.reset_mock()
# negative size will be adjusted as 0
utils.fallocate(0, -1, 0)
self.assertEqual(
[mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
@patch.object(os, 'fstatvfs')
@patch.object(utils, '_sys_fallocate', available=True, return_value=0)
@patch.object(utils, 'FALLOCATE_RESERVE', 0)
@patch.object(utils, 'FALLOCATE_IS_PERCENT', False)
@patch.object(utils, '_fallocate_enabled', True)
class TestFallocateReserve(unittest.TestCase):
def _statvfs_result(self, f_frsize, f_bavail):
# Only 3 values are relevant to us, so use zeros for the rest
f_blocks = 100
return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
0, 0, 0, 0, 0))
def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
utils.disable_fallocate()
utils.fallocate(123, 456)
sys_fallocate_mock.assert_not_called()
fstatvfs_mock.assert_not_called()
def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
utils.fallocate(123, 456)
fstatvfs_mock.assert_not_called()
self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
# of size 1024 free, so succeed
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
utils.fallocate(88, 1023)
def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
# of size 1024 free, so fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 1024)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
# of size 1024 free, so fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 1 << 30)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
% (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
sys_fallocate_mock.assert_not_called()
def test_enough_space_small_blocks(self, sys_fallocate_mock,
fstatvfs_mock):
# Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
# of size 512 free, so succeed
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
utils.fallocate(88, 1023)
def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
fstatvfs_mock):
# Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
# of size 512 free, so fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 1024)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
# Want 2048 bytes in reserve but have only 3 blocks of size 512, so
# allocating even 0 bytes fails
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('2048')
fstatvfs_mock.return_value = self._statvfs_result(512, 3)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 0)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
# Filesystem is empty, but our reserve is bigger than the
# filesystem, so any allocation will fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('9999999999999')
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
self.assertRaises(OSError, utils.fallocate, 88, 0)
sys_fallocate_mock.assert_not_called()
def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
# and file size is 2047, so succeed
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
utils.fallocate(88, 2047)
def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
# and file size is 2048, so fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('1%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 2048)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Filesystem is empty, but our reserve is the whole filesystem, so
# any allocation will fail
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value('100%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
with self.assertRaises(OSError) as catcher:
utils.fallocate(88, 0)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
@patch('ctypes.get_errno')
@patch.object(utils, '_sys_fallocate')
class TestPunchHole(unittest.TestCase):
def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
utils.punch_hole(123, 456, 789)
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 123)
self.assertEqual(
args[1], utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 456)
self.assertEqual(args[3].value, 789)
def test_error(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
get_errno_mock.return_value = errno.EISDIR
with self.assertRaises(OSError) as cm:
utils.punch_hole(123, 456, 789)
self.assertEqual(cm.exception.errno, errno.EISDIR)
def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
with self.assertRaises(ValueError):
utils.punch_hole(0, 1, -1)
with self.assertRaises(ValueError):
utils.punch_hole(0, 1 << 64, 1)
with self.assertRaises(ValueError):
utils.punch_hole(0, -1, 1)
with self.assertRaises(ValueError):
utils.punch_hole(0, 1, 0)
with self.assertRaises(ValueError):
utils.punch_hole(0, 1, 1 << 64)
self.assertEqual([], sys_fallocate_mock.mock_calls)
# sanity check
utils.punch_hole(0, 0, 1)
self.assertEqual(
[mock.call(
0, utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE,
mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
# Go confirm the ctypes values separately; apparently == doesn't
# work the way you'd expect with ctypes :-/
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
with self.assertRaises(OSError) as cm:
utils.punch_hole(123, 456, 789)
self.assertEqual(cm.exception.errno, errno.ENOTSUP)
class TestPunchHoleReally(unittest.TestCase):
def setUp(self):
if not utils._sys_fallocate.available:
raise unittest.SkipTest("utils._sys_fallocate not available")
def test_punch_a_hole(self):
with TemporaryFile() as tf:
tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
tf.flush()
# knock out the first half of the "y"s
utils.punch_hole(tf.fileno(), 64, 32)
tf.seek(0)
contents = tf.read(4096)
self.assertEqual(
contents,
b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
class TestWatchdog(unittest.TestCase): class TestWatchdog(unittest.TestCase):
def test_start_stop(self): def test_start_stop(self):
w = utils.Watchdog() w = utils.Watchdog()

432
test/unit/common/utils/test_libc.py
View File

@ -16,10 +16,8 @@
"""Tests for swift.common.utils.libc""" """Tests for swift.common.utils.libc"""
import ctypes import ctypes
import errno
import os import os
import platform import platform
import posix
import tempfile import tempfile
import unittest import unittest
@ -30,436 +28,6 @@ from swift.common.utils import libc
from test.debug_logger import debug_logger from test.debug_logger import debug_logger
@mock.patch('ctypes.get_errno')
@mock.patch.object(libc, '_sys_posix_fallocate')
@mock.patch.object(libc, '_sys_fallocate')
@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
class TestFallocate(unittest.TestCase):
def test_config_fallocate_value(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
fallocate_value, is_percent = libc.config_fallocate_value('10%')
self.assertEqual(fallocate_value, 10)
self.assertTrue(is_percent)
fallocate_value, is_percent = libc.config_fallocate_value('10')
self.assertEqual(fallocate_value, 10)
self.assertFalse(is_percent)
try:
fallocate_value, is_percent = libc.config_fallocate_value('ab%')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: ab% is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = libc.config_fallocate_value('ab')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: ab is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = libc.config_fallocate_value('1%%')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: 1%% is an invalid value for '
'fallocate_reserve.')
try:
fallocate_value, is_percent = libc.config_fallocate_value('10.0')
except ValueError as err:
exc = err
self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for '
'fallocate_reserve.')
fallocate_value, is_percent = libc.config_fallocate_value('10.5%')
self.assertEqual(fallocate_value, 10.5)
self.assertTrue(is_percent)
fallocate_value, is_percent = libc.config_fallocate_value('10.000%')
self.assertEqual(fallocate_value, 10.000)
self.assertTrue(is_percent)
def test_fallocate(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
libc.fallocate(1234, 5000 * 2 ** 20)
# We can't use sys_fallocate_mock.assert_called_once_with because no
# two ctypes.c_uint64 objects are equal even if their values are
# equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123).
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 0)
self.assertEqual(args[3].value, 5000 * 2 ** 20)
sys_posix_fallocate_mock.assert_not_called()
def test_fallocate_offset(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 3 * 2 ** 30)
self.assertEqual(args[3].value, 5000 * 2 ** 20)
sys_posix_fallocate_mock.assert_not_called()
def test_fallocate_fatal_error(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
get_errno_mock.return_value = errno.EIO
with self.assertRaises(OSError) as cm:
libc.fallocate(1234, 5000 * 2 ** 20)
self.assertEqual(cm.exception.errno, errno.EIO)
def test_fallocate_silent_errors(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL):
get_errno_mock.return_value = silent_error
try:
libc.fallocate(1234, 5678)
except OSError:
self.fail("fallocate() raised an error on %d", silent_error)
def test_posix_fallocate_fallback(self, sys_fallocate_mock,
sys_posix_fallocate_mock,
get_errno_mock):
sys_fallocate_mock.available = False
sys_fallocate_mock.side_effect = NotImplementedError
sys_posix_fallocate_mock.available = True
sys_posix_fallocate_mock.return_value = 0
libc.fallocate(1234, 567890)
sys_fallocate_mock.assert_not_called()
calls = sys_posix_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 3)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1].value, 0)
self.assertEqual(args[2].value, 567890)
def test_posix_fallocate_offset(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
sys_fallocate_mock.side_effect = NotImplementedError
sys_posix_fallocate_mock.available = True
sys_posix_fallocate_mock.return_value = 0
libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30)
calls = sys_posix_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 3)
self.assertEqual(args[0], 1234)
self.assertEqual(args[1].value, 3 * 2 ** 30)
self.assertEqual(args[2].value, 5000 * 2 ** 20)
sys_fallocate_mock.assert_not_called()
def test_no_fallocates_available(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
sys_posix_fallocate_mock.available = False
with mock.patch("logging.warning") as warning_mock, \
mock.patch.object(libc, "_fallocate_warned_about_missing",
False):
libc.fallocate(321, 654)
libc.fallocate(321, 654)
sys_fallocate_mock.assert_not_called()
sys_posix_fallocate_mock.assert_not_called()
get_errno_mock.assert_not_called()
self.assertEqual(len(warning_mock.mock_calls), 1)
def test_arg_bounds(self, sys_fallocate_mock,
sys_posix_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
with self.assertRaises(ValueError):
libc.fallocate(0, 1 << 64, 0)
with self.assertRaises(ValueError):
libc.fallocate(0, 0, -1)
with self.assertRaises(ValueError):
libc.fallocate(0, 0, 1 << 64)
self.assertEqual([], sys_fallocate_mock.mock_calls)
# sanity check
libc.fallocate(0, 0, 0)
self.assertEqual(
[mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
# Go confirm the ctypes values separately; apparently == doesn't
# work the way you'd expect with ctypes :-/
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
sys_fallocate_mock.reset_mock()
# negative size will be adjusted as 0
libc.fallocate(0, -1, 0)
self.assertEqual(
[mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0)
@mock.patch.object(os, 'fstatvfs')
@mock.patch.object(libc, '_sys_fallocate', available=True, return_value=0)
@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0)
@mock.patch.object(libc, 'FALLOCATE_IS_PERCENT', False)
@mock.patch.object(libc, '_fallocate_enabled', True)
class TestFallocateReserve(unittest.TestCase):
def _statvfs_result(self, f_frsize, f_bavail):
# Only 3 values are relevant to us, so use zeros for the rest
f_blocks = 100
return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail,
0, 0, 0, 0, 0))
def test_disabled(self, sys_fallocate_mock, fstatvfs_mock):
libc.disable_fallocate()
libc.fallocate(123, 456)
sys_fallocate_mock.assert_not_called()
fstatvfs_mock.assert_not_called()
def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock):
libc.fallocate(123, 456)
fstatvfs_mock.assert_not_called()
self.assertEqual(len(sys_fallocate_mock.mock_calls), 1)
def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks
# of size 1024 free, so succeed
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
libc.fallocate(88, 1023)
def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks
# of size 1024 free, so fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 1024)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks
# of size 1024 free, so fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(1024, 2)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 1 << 30)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail %g <= 1024'
% (errno.ENOSPC, ((2 * 1024) - (1 << 30))))
sys_fallocate_mock.assert_not_called()
def test_enough_space_small_blocks(self, sys_fallocate_mock,
fstatvfs_mock):
# Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks
# of size 512 free, so succeed
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
libc.fallocate(88, 1023)
def test_not_enough_space_small_blocks(self, sys_fallocate_mock,
fstatvfs_mock):
# Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks
# of size 512 free, so fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1024')
fstatvfs_mock.return_value = self._statvfs_result(512, 4)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 1024)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock):
# Want 2048 bytes in reserve but have only 3 blocks of size 512, so
# allocating even 0 bytes fails
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('2048')
fstatvfs_mock.return_value = self._statvfs_result(512, 3)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 0)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock):
# Filesystem is empty, but our reserve is bigger than the
# filesystem, so any allocation will fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('9999999999999')
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
self.assertRaises(OSError, libc.fallocate, 88, 0)
sys_fallocate_mock.assert_not_called()
def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
# and file size is 2047, so succeed
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
libc.fallocate(88, 2047)
def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Want 1% reserved, filesystem has 3/100 blocks of size 1024 free
# and file size is 2048, so fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('1%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 3)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 2048)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 1 <= 1'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock):
# Filesystem is empty, but our reserve is the whole filesystem, so
# any allocation will fail
libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \
libc.config_fallocate_value('100%')
fstatvfs_mock.return_value = self._statvfs_result(1024, 100)
with self.assertRaises(OSError) as catcher:
libc.fallocate(88, 0)
self.assertEqual(
str(catcher.exception),
'[Errno %d] FALLOCATE_RESERVE fail 100 <= 100'
% errno.ENOSPC)
sys_fallocate_mock.assert_not_called()
@mock.patch('ctypes.get_errno')
@mock.patch.object(libc, '_sys_fallocate')
class TestPunchHole(unittest.TestCase):
def test_punch_hole(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
libc.punch_hole(123, 456, 789)
calls = sys_fallocate_mock.mock_calls
self.assertEqual(len(calls), 1)
args = calls[0][1]
self.assertEqual(len(args), 4)
self.assertEqual(args[0], 123)
self.assertEqual(
args[1], libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE)
self.assertEqual(args[2].value, 456)
self.assertEqual(args[3].value, 789)
def test_error(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = -1
get_errno_mock.return_value = errno.EISDIR
with self.assertRaises(OSError) as cm:
libc.punch_hole(123, 456, 789)
self.assertEqual(cm.exception.errno, errno.EISDIR)
def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = True
sys_fallocate_mock.return_value = 0
with self.assertRaises(ValueError):
libc.punch_hole(0, 1, -1)
with self.assertRaises(ValueError):
libc.punch_hole(0, 1 << 64, 1)
with self.assertRaises(ValueError):
libc.punch_hole(0, -1, 1)
with self.assertRaises(ValueError):
libc.punch_hole(0, 1, 0)
with self.assertRaises(ValueError):
libc.punch_hole(0, 1, 1 << 64)
self.assertEqual([], sys_fallocate_mock.mock_calls)
# sanity check
libc.punch_hole(0, 0, 1)
self.assertEqual(
[mock.call(
0, libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE,
mock.ANY, mock.ANY)],
sys_fallocate_mock.mock_calls)
# Go confirm the ctypes values separately; apparently == doesn't
# work the way you'd expect with ctypes :-/
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0)
self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1)
def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock):
sys_fallocate_mock.available = False
with self.assertRaises(OSError) as cm:
libc.punch_hole(123, 456, 789)
self.assertEqual(cm.exception.errno, errno.ENOTSUP)
class TestPunchHoleReally(unittest.TestCase):
def setUp(self):
if not libc._sys_fallocate.available:
raise unittest.SkipTest("libc._sys_fallocate not available")
def test_punch_a_hole(self):
with tempfile.TemporaryFile() as tf:
tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64)
tf.flush()
# knock out the first half of the "y"s
libc.punch_hole(tf.fileno(), 64, 32)
tf.seek(0)
contents = tf.read(4096)
self.assertEqual(
contents,
b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64)
class Test_LibcWrapper(unittest.TestCase): class Test_LibcWrapper(unittest.TestCase):
def test_available_function(self): def test_available_function(self):
# This should pretty much always exist # This should pretty much always exist

3
test/unit/obj/test_diskfile.py
View File

@ -50,7 +50,6 @@ from test.unit import (mock as unit_mock, temptree, mock_check_drive,
encode_frag_archive_bodies, skip_if_no_xattrs) encode_frag_archive_bodies, skip_if_no_xattrs)
from swift.obj import diskfile from swift.obj import diskfile
from swift.common import utils from swift.common import utils
from swift.common.utils import libc
from swift.common.utils import hash_path, mkdirs, Timestamp, \ from swift.common.utils import hash_path, mkdirs, Timestamp, \
encode_timestamps, O_TMPFILE, md5 as _md5 encode_timestamps, O_TMPFILE, md5 as _md5
from swift.common import ring from swift.common import ring
@ -4989,7 +4988,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
# This is a horrible hack so you can run this test in isolation. # This is a horrible hack so you can run this test in isolation.
# Some of the ctypes machinery calls os.close(), and that runs afoul # Some of the ctypes machinery calls os.close(), and that runs afoul
# of our mock. # of our mock.
with mock.patch.object(libc, '_sys_fallocate', None): with mock.patch.object(utils, '_sys_fallocate', None):
utils.disable_fallocate() utils.disable_fallocate()
df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc',