diff --git a/swift/common/utils/__init__.py b/swift/common/utils/__init__.py index 90fd5ecf63..a8040ef21e 100644 --- a/swift/common/utils/__init__.py +++ b/swift/common/utils/__init__.py @@ -39,6 +39,8 @@ import functools import email.parser from random import random, shuffle from contextlib import contextmanager, closing +import ctypes +import ctypes.util from optparse import OptionParser import traceback import warnings @@ -94,13 +96,10 @@ from swift.common.registry import register_swift_info, get_swift_info # noqa from swift.common.utils.libc import ( # noqa F_SETPIPE_SZ, load_libc_function, - config_fallocate_value, - disable_fallocate, - fallocate, - punch_hole, drop_buffer_cache, get_md5_socket, modify_priority, + _LibcWrapper, ) from swift.common.utils.timestamp import ( # noqa NORMAL_FORMAT, @@ -129,6 +128,21 @@ import logging NOTICE = 25 +# These are lazily pulled from libc elsewhere +_sys_fallocate = None + +# If set to non-zero, fallocate routines will fail based on free space +# available being at or below this amount, in bytes. +FALLOCATE_RESERVE = 0 +# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or +# the number of bytes (False). +FALLOCATE_IS_PERCENT = False + +# from /usr/include/linux/falloc.h +FALLOC_FL_KEEP_SIZE = 1 +FALLOC_FL_PUNCH_HOLE = 2 + + # Used by hash_path to offer a bit more security when generating hashes for # paths. It simply appends this value to all paths; guessing the hash a path # will end up with would also require knowing this suffix. @@ -137,6 +151,10 @@ HASH_PATH_PREFIX = b'' SWIFT_CONF_FILE = '/etc/swift/swift.conf' +# These constants are Linux-specific, and Python doesn't seem to know +# about them. We ask anyway just in case that ever gets fixed. +# +# The values were copied from the Linux 3.x kernel headers. O_TMPFILE = getattr(os, 'O_TMPFILE', 0o20000000 | os.O_DIRECTORY) MD5_OF_EMPTY_STRING = 'd41d8cd98f00b204e9800998ecf8427e' @@ -672,6 +690,25 @@ def get_trans_id_time(trans_id): return None +def config_fallocate_value(reserve_value): + """ + Returns fallocate reserve_value as an int or float. + Returns is_percent as a boolean. + Returns a ValueError on invalid fallocate value. + """ + try: + if str(reserve_value[-1:]) == '%': + reserve_value = float(reserve_value[:-1]) + is_percent = True + else: + reserve_value = int(reserve_value) + is_percent = False + except ValueError: + raise ValueError('Error: %s is an invalid value for fallocate' + '_reserve.' % reserve_value) + return reserve_value, is_percent + + class FileLikeIter(object): def __init__(self, iterable): @@ -822,6 +859,116 @@ def fs_has_free_space(fs_path, space_needed, is_percent): return free_bytes >= space_needed +_fallocate_enabled = True +_fallocate_warned_about_missing = False +_sys_fallocate = _LibcWrapper('fallocate') +_sys_posix_fallocate = _LibcWrapper('posix_fallocate') + + +def disable_fallocate(): + global _fallocate_enabled + _fallocate_enabled = False + + +def fallocate(fd, size, offset=0): + """ + Pre-allocate disk space for a file. + + This function can be disabled by calling disable_fallocate(). If no + suitable C function is available in libc, this function is a no-op. + + :param fd: file descriptor + :param size: size to allocate (in bytes) + """ + global _fallocate_enabled + if not _fallocate_enabled: + return + + if size < 0: + size = 0 # Done historically; not really sure why + if size >= (1 << 63): + raise ValueError('size must be less than 2 ** 63') + if offset < 0: + raise ValueError('offset must be non-negative') + if offset >= (1 << 63): + raise ValueError('offset must be less than 2 ** 63') + + # Make sure there's some (configurable) amount of free space in + # addition to the number of bytes we're allocating. + if FALLOCATE_RESERVE: + st = os.fstatvfs(fd) + free = st.f_frsize * st.f_bavail - size + if FALLOCATE_IS_PERCENT: + free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100 + if float(free) <= float(FALLOCATE_RESERVE): + raise OSError( + errno.ENOSPC, + 'FALLOCATE_RESERVE fail %g <= %g' % + (free, FALLOCATE_RESERVE)) + + if _sys_fallocate.available: + # Parameters are (fd, mode, offset, length). + # + # mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without + # affecting the reported file size). + ret = _sys_fallocate( + fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset), + ctypes.c_uint64(size)) + err = ctypes.get_errno() + elif _sys_posix_fallocate.available: + # Parameters are (fd, offset, length). + ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset), + ctypes.c_uint64(size)) + err = ctypes.get_errno() + else: + # No suitable fallocate-like function is in our libc. Warn about it, + # but just once per process, and then do nothing. + global _fallocate_warned_about_missing + if not _fallocate_warned_about_missing: + logging.warning("Unable to locate fallocate, posix_fallocate in " + "libc. Leaving as a no-op.") + _fallocate_warned_about_missing = True + return + + if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP, + errno.EINVAL): + raise OSError(err, 'Unable to fallocate(%s)' % size) + + +def punch_hole(fd, offset, length): + """ + De-allocate disk space in the middle of a file. + + :param fd: file descriptor + :param offset: index of first byte to de-allocate + :param length: number of bytes to de-allocate + """ + if offset < 0: + raise ValueError('offset must be non-negative') + if offset >= (1 << 63): + raise ValueError('offset must be less than 2 ** 63') + if length <= 0: + raise ValueError('length must be positive') + if length >= (1 << 63): + raise ValueError('length must be less than 2 ** 63') + + if _sys_fallocate.available: + # Parameters are (fd, mode, offset, length). + ret = _sys_fallocate( + fd, + FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, + ctypes.c_uint64(offset), + ctypes.c_uint64(length)) + err = ctypes.get_errno() + if ret and err: + mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE" + raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % ( + fd, mode_str, offset, length)) + else: + raise OSError(errno.ENOTSUP, + 'No suitable C function found for hole punching') + + def fsync(fd): """ Sync modified file data and metadata to disk. diff --git a/swift/common/utils/libc.py b/swift/common/utils/libc.py index df21790209..94571157d1 100644 --- a/swift/common/utils/libc.py +++ b/swift/common/utils/libc.py @@ -17,7 +17,6 @@ import ctypes import ctypes.util -import errno import fcntl import logging import os @@ -26,7 +25,6 @@ import socket # These are lazily pulled from libc elsewhere -_sys_fallocate = None _posix_fadvise = None _libc_socket = None _libc_bind = None @@ -36,17 +34,6 @@ _libc_setpriority = None # see man -s 2 syscall _posix_syscall = None -# If set to non-zero, fallocate routines will fail based on free space -# available being at or below this amount, in bytes. -FALLOCATE_RESERVE = 0 -# Indicates if FALLOCATE_RESERVE is the percentage of free space (True) or -# the number of bytes (False). -FALLOCATE_IS_PERCENT = False - -# from /usr/include/linux/falloc.h -FALLOC_FL_KEEP_SIZE = 1 -FALLOC_FL_PUNCH_HOLE = 2 - # from /usr/src/linux-headers-*/include/uapi/linux/resource.h PRIO_PROCESS = 0 @@ -191,135 +178,6 @@ class _LibcWrapper(object): "No function %r found in libc" % self._func_name) -def config_fallocate_value(reserve_value): - """ - Returns fallocate reserve_value as an int or float. - Returns is_percent as a boolean. - Returns a ValueError on invalid fallocate value. - """ - try: - if str(reserve_value[-1:]) == '%': - reserve_value = float(reserve_value[:-1]) - is_percent = True - else: - reserve_value = int(reserve_value) - is_percent = False - except ValueError: - raise ValueError('Error: %s is an invalid value for fallocate' - '_reserve.' % reserve_value) - return reserve_value, is_percent - - -_fallocate_enabled = True -_fallocate_warned_about_missing = False -_sys_fallocate = _LibcWrapper('fallocate') -_sys_posix_fallocate = _LibcWrapper('posix_fallocate') - - -def disable_fallocate(): - global _fallocate_enabled - _fallocate_enabled = False - - -def fallocate(fd, size, offset=0): - """ - Pre-allocate disk space for a file. - - This function can be disabled by calling disable_fallocate(). If no - suitable C function is available in libc, this function is a no-op. - - :param fd: file descriptor - :param size: size to allocate (in bytes) - """ - global _fallocate_enabled - if not _fallocate_enabled: - return - - if size < 0: - size = 0 # Done historically; not really sure why - if size >= (1 << 63): - raise ValueError('size must be less than 2 ** 63') - if offset < 0: - raise ValueError('offset must be non-negative') - if offset >= (1 << 63): - raise ValueError('offset must be less than 2 ** 63') - - # Make sure there's some (configurable) amount of free space in - # addition to the number of bytes we're allocating. - if FALLOCATE_RESERVE: - st = os.fstatvfs(fd) - free = st.f_frsize * st.f_bavail - size - if FALLOCATE_IS_PERCENT: - free = (float(free) / float(st.f_frsize * st.f_blocks)) * 100 - if float(free) <= float(FALLOCATE_RESERVE): - raise OSError( - errno.ENOSPC, - 'FALLOCATE_RESERVE fail %g <= %g' % - (free, FALLOCATE_RESERVE)) - - if _sys_fallocate.available: - # Parameters are (fd, mode, offset, length). - # - # mode=FALLOC_FL_KEEP_SIZE pre-allocates invisibly (without - # affecting the reported file size). - ret = _sys_fallocate( - fd, FALLOC_FL_KEEP_SIZE, ctypes.c_uint64(offset), - ctypes.c_uint64(size)) - err = ctypes.get_errno() - elif _sys_posix_fallocate.available: - # Parameters are (fd, offset, length). - ret = _sys_posix_fallocate(fd, ctypes.c_uint64(offset), - ctypes.c_uint64(size)) - err = ctypes.get_errno() - else: - # No suitable fallocate-like function is in our libc. Warn about it, - # but just once per process, and then do nothing. - global _fallocate_warned_about_missing - if not _fallocate_warned_about_missing: - logging.warning("Unable to locate fallocate, posix_fallocate in " - "libc. Leaving as a no-op.") - _fallocate_warned_about_missing = True - return - - if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP, - errno.EINVAL): - raise OSError(err, 'Unable to fallocate(%s)' % size) - - -def punch_hole(fd, offset, length): - """ - De-allocate disk space in the middle of a file. - - :param fd: file descriptor - :param offset: index of first byte to de-allocate - :param length: number of bytes to de-allocate - """ - if offset < 0: - raise ValueError('offset must be non-negative') - if offset >= (1 << 63): - raise ValueError('offset must be less than 2 ** 63') - if length <= 0: - raise ValueError('length must be positive') - if length >= (1 << 63): - raise ValueError('length must be less than 2 ** 63') - - if _sys_fallocate.available: - # Parameters are (fd, mode, offset, length). - ret = _sys_fallocate( - fd, - FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, - ctypes.c_uint64(offset), - ctypes.c_uint64(length)) - err = ctypes.get_errno() - if ret and err: - mode_str = "FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE" - raise OSError(err, "Unable to fallocate(%d, %s, %d, %d)" % ( - fd, mode_str, offset, length)) - else: - raise OSError(errno.ENOTSUP, - 'No suitable C function found for hole punching') - - def drop_buffer_cache(fd, offset, length): """ Drop 'buffer' cache for the given range of the given file. diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 8a58b66478..3c320ddc4f 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2712,6 +2712,44 @@ cluster_dfw1 = http://dfw1.host/v1/ ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright') self.assertIsNone(ts) + def test_config_fallocate_value(self): + fallocate_value, is_percent = utils.config_fallocate_value('10%') + self.assertEqual(fallocate_value, 10) + self.assertTrue(is_percent) + fallocate_value, is_percent = utils.config_fallocate_value('10') + self.assertEqual(fallocate_value, 10) + self.assertFalse(is_percent) + try: + fallocate_value, is_percent = utils.config_fallocate_value('ab%') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: ab% is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('ab') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: ab is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('1%%') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: 1%% is an invalid value for ' + 'fallocate_reserve.') + try: + fallocate_value, is_percent = utils.config_fallocate_value('10.0') + except ValueError as err: + exc = err + self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for ' + 'fallocate_reserve.') + fallocate_value, is_percent = utils.config_fallocate_value('10.5%') + self.assertEqual(fallocate_value, 10.5) + self.assertTrue(is_percent) + fallocate_value, is_percent = utils.config_fallocate_value('10.000%') + self.assertEqual(fallocate_value, 10.000) + self.assertTrue(is_percent) + def test_lock_file(self): flags = os.O_CREAT | os.O_RDWR with NamedTemporaryFile(delete=False) as nt: @@ -8785,6 +8823,397 @@ class TestShardRangeList(unittest.TestCase): do_test([utils.ShardRange.ACTIVE])) +@patch('ctypes.get_errno') +@patch.object(utils, '_sys_posix_fallocate') +@patch.object(utils, '_sys_fallocate') +@patch.object(utils, 'FALLOCATE_RESERVE', 0) +class TestFallocate(unittest.TestCase): + def test_fallocate(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = 0 + + utils.fallocate(1234, 5000 * 2 ** 20) + + # We can't use sys_fallocate_mock.assert_called_once_with because no + # two ctypes.c_uint64 objects are equal even if their values are + # equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123). + calls = sys_fallocate_mock.mock_calls + self.assertEqual(len(calls), 1) + args = calls[0][1] + self.assertEqual(len(args), 4) + self.assertEqual(args[0], 1234) + self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE) + self.assertEqual(args[2].value, 0) + self.assertEqual(args[3].value, 5000 * 2 ** 20) + + sys_posix_fallocate_mock.assert_not_called() + + def test_fallocate_offset(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = 0 + + utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30) + calls = sys_fallocate_mock.mock_calls + self.assertEqual(len(calls), 1) + args = calls[0][1] + self.assertEqual(len(args), 4) + self.assertEqual(args[0], 1234) + self.assertEqual(args[1], utils.FALLOC_FL_KEEP_SIZE) + self.assertEqual(args[2].value, 3 * 2 ** 30) + self.assertEqual(args[3].value, 5000 * 2 ** 20) + + sys_posix_fallocate_mock.assert_not_called() + + def test_fallocate_fatal_error(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = -1 + get_errno_mock.return_value = errno.EIO + + with self.assertRaises(OSError) as cm: + utils.fallocate(1234, 5000 * 2 ** 20) + self.assertEqual(cm.exception.errno, errno.EIO) + + def test_fallocate_silent_errors(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = -1 + + for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL): + get_errno_mock.return_value = silent_error + try: + utils.fallocate(1234, 5678) + except OSError: + self.fail("fallocate() raised an error on %d", silent_error) + + def test_posix_fallocate_fallback(self, sys_fallocate_mock, + sys_posix_fallocate_mock, + get_errno_mock): + sys_fallocate_mock.available = False + sys_fallocate_mock.side_effect = NotImplementedError + + sys_posix_fallocate_mock.available = True + sys_posix_fallocate_mock.return_value = 0 + + utils.fallocate(1234, 567890) + sys_fallocate_mock.assert_not_called() + + calls = sys_posix_fallocate_mock.mock_calls + self.assertEqual(len(calls), 1) + args = calls[0][1] + self.assertEqual(len(args), 3) + self.assertEqual(args[0], 1234) + self.assertEqual(args[1].value, 0) + self.assertEqual(args[2].value, 567890) + + def test_posix_fallocate_offset(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = False + sys_fallocate_mock.side_effect = NotImplementedError + + sys_posix_fallocate_mock.available = True + sys_posix_fallocate_mock.return_value = 0 + + utils.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30) + calls = sys_posix_fallocate_mock.mock_calls + self.assertEqual(len(calls), 1) + args = calls[0][1] + self.assertEqual(len(args), 3) + self.assertEqual(args[0], 1234) + self.assertEqual(args[1].value, 3 * 2 ** 30) + self.assertEqual(args[2].value, 5000 * 2 ** 20) + + sys_fallocate_mock.assert_not_called() + + def test_no_fallocates_available(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = False + sys_posix_fallocate_mock.available = False + + with mock.patch("logging.warning") as warning_mock, \ + mock.patch.object(utils, "_fallocate_warned_about_missing", + False): + utils.fallocate(321, 654) + utils.fallocate(321, 654) + + sys_fallocate_mock.assert_not_called() + sys_posix_fallocate_mock.assert_not_called() + get_errno_mock.assert_not_called() + + self.assertEqual(len(warning_mock.mock_calls), 1) + + def test_arg_bounds(self, sys_fallocate_mock, + sys_posix_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = 0 + with self.assertRaises(ValueError): + utils.fallocate(0, 1 << 64, 0) + with self.assertRaises(ValueError): + utils.fallocate(0, 0, -1) + with self.assertRaises(ValueError): + utils.fallocate(0, 0, 1 << 64) + self.assertEqual([], sys_fallocate_mock.mock_calls) + # sanity check + utils.fallocate(0, 0, 0) + self.assertEqual( + [mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)], + sys_fallocate_mock.mock_calls) + # Go confirm the ctypes values separately; apparently == doesn't + # work the way you'd expect with ctypes :-/ + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0) + sys_fallocate_mock.reset_mock() + + # negative size will be adjusted as 0 + utils.fallocate(0, -1, 0) + self.assertEqual( + [mock.call(0, utils.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)], + sys_fallocate_mock.mock_calls) + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0) + + +@patch.object(os, 'fstatvfs') +@patch.object(utils, '_sys_fallocate', available=True, return_value=0) +@patch.object(utils, 'FALLOCATE_RESERVE', 0) +@patch.object(utils, 'FALLOCATE_IS_PERCENT', False) +@patch.object(utils, '_fallocate_enabled', True) +class TestFallocateReserve(unittest.TestCase): + def _statvfs_result(self, f_frsize, f_bavail): + # Only 3 values are relevant to us, so use zeros for the rest + f_blocks = 100 + return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail, + 0, 0, 0, 0, 0)) + + def test_disabled(self, sys_fallocate_mock, fstatvfs_mock): + utils.disable_fallocate() + utils.fallocate(123, 456) + + sys_fallocate_mock.assert_not_called() + fstatvfs_mock.assert_not_called() + + def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock): + utils.fallocate(123, 456) + + fstatvfs_mock.assert_not_called() + self.assertEqual(len(sys_fallocate_mock.mock_calls), 1) + + def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock): + # Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks + # of size 1024 free, so succeed + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 2) + utils.fallocate(88, 1023) + + def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock): + # Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks + # of size 1024 free, so fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 2) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 1024) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' + % errno.ENOSPC) + sys_fallocate_mock.assert_not_called() + + def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock): + # Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks + # of size 1024 free, so fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 2) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 1 << 30) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail %g <= 1024' + % (errno.ENOSPC, ((2 * 1024) - (1 << 30)))) + sys_fallocate_mock.assert_not_called() + + def test_enough_space_small_blocks(self, sys_fallocate_mock, + fstatvfs_mock): + # Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks + # of size 512 free, so succeed + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') + + fstatvfs_mock.return_value = self._statvfs_result(512, 4) + utils.fallocate(88, 1023) + + def test_not_enough_space_small_blocks(self, sys_fallocate_mock, + fstatvfs_mock): + # Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks + # of size 512 free, so fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1024') + + fstatvfs_mock.return_value = self._statvfs_result(512, 4) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 1024) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' + % errno.ENOSPC) + sys_fallocate_mock.assert_not_called() + + def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock): + # Want 2048 bytes in reserve but have only 3 blocks of size 512, so + # allocating even 0 bytes fails + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('2048') + + fstatvfs_mock.return_value = self._statvfs_result(512, 3) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 0) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048' + % errno.ENOSPC) + sys_fallocate_mock.assert_not_called() + + def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock): + # Filesystem is empty, but our reserve is bigger than the + # filesystem, so any allocation will fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('9999999999999') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 100) + self.assertRaises(OSError, utils.fallocate, 88, 0) + sys_fallocate_mock.assert_not_called() + + def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock): + # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free + # and file size is 2047, so succeed + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1%') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 3) + utils.fallocate(88, 2047) + + def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock): + # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free + # and file size is 2048, so fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('1%') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 3) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 2048) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 1 <= 1' + % errno.ENOSPC) + sys_fallocate_mock.assert_not_called() + + def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock): + # Filesystem is empty, but our reserve is the whole filesystem, so + # any allocation will fail + utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \ + utils.config_fallocate_value('100%') + + fstatvfs_mock.return_value = self._statvfs_result(1024, 100) + with self.assertRaises(OSError) as catcher: + utils.fallocate(88, 0) + self.assertEqual( + str(catcher.exception), + '[Errno %d] FALLOCATE_RESERVE fail 100 <= 100' + % errno.ENOSPC) + sys_fallocate_mock.assert_not_called() + + +@patch('ctypes.get_errno') +@patch.object(utils, '_sys_fallocate') +class TestPunchHole(unittest.TestCase): + def test_punch_hole(self, sys_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = 0 + + utils.punch_hole(123, 456, 789) + + calls = sys_fallocate_mock.mock_calls + self.assertEqual(len(calls), 1) + args = calls[0][1] + self.assertEqual(len(args), 4) + self.assertEqual(args[0], 123) + self.assertEqual( + args[1], utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE) + self.assertEqual(args[2].value, 456) + self.assertEqual(args[3].value, 789) + + def test_error(self, sys_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = -1 + get_errno_mock.return_value = errno.EISDIR + + with self.assertRaises(OSError) as cm: + utils.punch_hole(123, 456, 789) + self.assertEqual(cm.exception.errno, errno.EISDIR) + + def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = True + sys_fallocate_mock.return_value = 0 + + with self.assertRaises(ValueError): + utils.punch_hole(0, 1, -1) + with self.assertRaises(ValueError): + utils.punch_hole(0, 1 << 64, 1) + with self.assertRaises(ValueError): + utils.punch_hole(0, -1, 1) + with self.assertRaises(ValueError): + utils.punch_hole(0, 1, 0) + with self.assertRaises(ValueError): + utils.punch_hole(0, 1, 1 << 64) + self.assertEqual([], sys_fallocate_mock.mock_calls) + + # sanity check + utils.punch_hole(0, 0, 1) + self.assertEqual( + [mock.call( + 0, utils.FALLOC_FL_PUNCH_HOLE | utils.FALLOC_FL_KEEP_SIZE, + mock.ANY, mock.ANY)], + sys_fallocate_mock.mock_calls) + # Go confirm the ctypes values separately; apparently == doesn't + # work the way you'd expect with ctypes :-/ + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) + self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1) + + def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock): + sys_fallocate_mock.available = False + + with self.assertRaises(OSError) as cm: + utils.punch_hole(123, 456, 789) + self.assertEqual(cm.exception.errno, errno.ENOTSUP) + + +class TestPunchHoleReally(unittest.TestCase): + def setUp(self): + if not utils._sys_fallocate.available: + raise unittest.SkipTest("utils._sys_fallocate not available") + + def test_punch_a_hole(self): + with TemporaryFile() as tf: + tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64) + tf.flush() + + # knock out the first half of the "y"s + utils.punch_hole(tf.fileno(), 64, 32) + + tf.seek(0) + contents = tf.read(4096) + self.assertEqual( + contents, + b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64) + + class TestWatchdog(unittest.TestCase): def test_start_stop(self): w = utils.Watchdog() diff --git a/test/unit/common/utils/test_libc.py b/test/unit/common/utils/test_libc.py index 5357ce34d7..40fa7572cd 100644 --- a/test/unit/common/utils/test_libc.py +++ b/test/unit/common/utils/test_libc.py @@ -16,10 +16,8 @@ """Tests for swift.common.utils.libc""" import ctypes -import errno import os import platform -import posix import tempfile import unittest @@ -30,436 +28,6 @@ from swift.common.utils import libc from test.debug_logger import debug_logger -@mock.patch('ctypes.get_errno') -@mock.patch.object(libc, '_sys_posix_fallocate') -@mock.patch.object(libc, '_sys_fallocate') -@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0) -class TestFallocate(unittest.TestCase): - def test_config_fallocate_value(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - fallocate_value, is_percent = libc.config_fallocate_value('10%') - self.assertEqual(fallocate_value, 10) - self.assertTrue(is_percent) - fallocate_value, is_percent = libc.config_fallocate_value('10') - self.assertEqual(fallocate_value, 10) - self.assertFalse(is_percent) - try: - fallocate_value, is_percent = libc.config_fallocate_value('ab%') - except ValueError as err: - exc = err - self.assertEqual(str(exc), 'Error: ab% is an invalid value for ' - 'fallocate_reserve.') - try: - fallocate_value, is_percent = libc.config_fallocate_value('ab') - except ValueError as err: - exc = err - self.assertEqual(str(exc), 'Error: ab is an invalid value for ' - 'fallocate_reserve.') - try: - fallocate_value, is_percent = libc.config_fallocate_value('1%%') - except ValueError as err: - exc = err - self.assertEqual(str(exc), 'Error: 1%% is an invalid value for ' - 'fallocate_reserve.') - try: - fallocate_value, is_percent = libc.config_fallocate_value('10.0') - except ValueError as err: - exc = err - self.assertEqual(str(exc), 'Error: 10.0 is an invalid value for ' - 'fallocate_reserve.') - fallocate_value, is_percent = libc.config_fallocate_value('10.5%') - self.assertEqual(fallocate_value, 10.5) - self.assertTrue(is_percent) - fallocate_value, is_percent = libc.config_fallocate_value('10.000%') - self.assertEqual(fallocate_value, 10.000) - self.assertTrue(is_percent) - - def test_fallocate(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = 0 - - libc.fallocate(1234, 5000 * 2 ** 20) - - # We can't use sys_fallocate_mock.assert_called_once_with because no - # two ctypes.c_uint64 objects are equal even if their values are - # equal. Yes, ctypes.c_uint64(123) != ctypes.c_uint64(123). - calls = sys_fallocate_mock.mock_calls - self.assertEqual(len(calls), 1) - args = calls[0][1] - self.assertEqual(len(args), 4) - self.assertEqual(args[0], 1234) - self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE) - self.assertEqual(args[2].value, 0) - self.assertEqual(args[3].value, 5000 * 2 ** 20) - - sys_posix_fallocate_mock.assert_not_called() - - def test_fallocate_offset(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = 0 - - libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30) - calls = sys_fallocate_mock.mock_calls - self.assertEqual(len(calls), 1) - args = calls[0][1] - self.assertEqual(len(args), 4) - self.assertEqual(args[0], 1234) - self.assertEqual(args[1], libc.FALLOC_FL_KEEP_SIZE) - self.assertEqual(args[2].value, 3 * 2 ** 30) - self.assertEqual(args[3].value, 5000 * 2 ** 20) - - sys_posix_fallocate_mock.assert_not_called() - - def test_fallocate_fatal_error(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = -1 - get_errno_mock.return_value = errno.EIO - - with self.assertRaises(OSError) as cm: - libc.fallocate(1234, 5000 * 2 ** 20) - self.assertEqual(cm.exception.errno, errno.EIO) - - def test_fallocate_silent_errors(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = -1 - - for silent_error in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL): - get_errno_mock.return_value = silent_error - try: - libc.fallocate(1234, 5678) - except OSError: - self.fail("fallocate() raised an error on %d", silent_error) - - def test_posix_fallocate_fallback(self, sys_fallocate_mock, - sys_posix_fallocate_mock, - get_errno_mock): - sys_fallocate_mock.available = False - sys_fallocate_mock.side_effect = NotImplementedError - - sys_posix_fallocate_mock.available = True - sys_posix_fallocate_mock.return_value = 0 - - libc.fallocate(1234, 567890) - sys_fallocate_mock.assert_not_called() - - calls = sys_posix_fallocate_mock.mock_calls - self.assertEqual(len(calls), 1) - args = calls[0][1] - self.assertEqual(len(args), 3) - self.assertEqual(args[0], 1234) - self.assertEqual(args[1].value, 0) - self.assertEqual(args[2].value, 567890) - - def test_posix_fallocate_offset(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = False - sys_fallocate_mock.side_effect = NotImplementedError - - sys_posix_fallocate_mock.available = True - sys_posix_fallocate_mock.return_value = 0 - - libc.fallocate(1234, 5000 * 2 ** 20, offset=3 * 2 ** 30) - calls = sys_posix_fallocate_mock.mock_calls - self.assertEqual(len(calls), 1) - args = calls[0][1] - self.assertEqual(len(args), 3) - self.assertEqual(args[0], 1234) - self.assertEqual(args[1].value, 3 * 2 ** 30) - self.assertEqual(args[2].value, 5000 * 2 ** 20) - - sys_fallocate_mock.assert_not_called() - - def test_no_fallocates_available(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = False - sys_posix_fallocate_mock.available = False - - with mock.patch("logging.warning") as warning_mock, \ - mock.patch.object(libc, "_fallocate_warned_about_missing", - False): - libc.fallocate(321, 654) - libc.fallocate(321, 654) - - sys_fallocate_mock.assert_not_called() - sys_posix_fallocate_mock.assert_not_called() - get_errno_mock.assert_not_called() - - self.assertEqual(len(warning_mock.mock_calls), 1) - - def test_arg_bounds(self, sys_fallocate_mock, - sys_posix_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = 0 - with self.assertRaises(ValueError): - libc.fallocate(0, 1 << 64, 0) - with self.assertRaises(ValueError): - libc.fallocate(0, 0, -1) - with self.assertRaises(ValueError): - libc.fallocate(0, 0, 1 << 64) - self.assertEqual([], sys_fallocate_mock.mock_calls) - # sanity check - libc.fallocate(0, 0, 0) - self.assertEqual( - [mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)], - sys_fallocate_mock.mock_calls) - # Go confirm the ctypes values separately; apparently == doesn't - # work the way you'd expect with ctypes :-/ - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0) - sys_fallocate_mock.reset_mock() - - # negative size will be adjusted as 0 - libc.fallocate(0, -1, 0) - self.assertEqual( - [mock.call(0, libc.FALLOC_FL_KEEP_SIZE, mock.ANY, mock.ANY)], - sys_fallocate_mock.mock_calls) - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 0) - - -@mock.patch.object(os, 'fstatvfs') -@mock.patch.object(libc, '_sys_fallocate', available=True, return_value=0) -@mock.patch.object(libc, 'FALLOCATE_RESERVE', 0) -@mock.patch.object(libc, 'FALLOCATE_IS_PERCENT', False) -@mock.patch.object(libc, '_fallocate_enabled', True) -class TestFallocateReserve(unittest.TestCase): - def _statvfs_result(self, f_frsize, f_bavail): - # Only 3 values are relevant to us, so use zeros for the rest - f_blocks = 100 - return posix.statvfs_result((0, f_frsize, f_blocks, 0, f_bavail, - 0, 0, 0, 0, 0)) - - def test_disabled(self, sys_fallocate_mock, fstatvfs_mock): - libc.disable_fallocate() - libc.fallocate(123, 456) - - sys_fallocate_mock.assert_not_called() - fstatvfs_mock.assert_not_called() - - def test_zero_reserve(self, sys_fallocate_mock, fstatvfs_mock): - libc.fallocate(123, 456) - - fstatvfs_mock.assert_not_called() - self.assertEqual(len(sys_fallocate_mock.mock_calls), 1) - - def test_enough_space(self, sys_fallocate_mock, fstatvfs_mock): - # Want 1024 bytes in reserve plus 1023 allocated, and have 2 blocks - # of size 1024 free, so succeed - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1024') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 2) - libc.fallocate(88, 1023) - - def test_not_enough_space(self, sys_fallocate_mock, fstatvfs_mock): - # Want 1024 bytes in reserve plus 1024 allocated, and have 2 blocks - # of size 1024 free, so fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1024') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 2) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 1024) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' - % errno.ENOSPC) - sys_fallocate_mock.assert_not_called() - - def test_not_enough_space_large(self, sys_fallocate_mock, fstatvfs_mock): - # Want 1024 bytes in reserve plus 1GB allocated, and have 2 blocks - # of size 1024 free, so fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1024') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 2) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 1 << 30) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail %g <= 1024' - % (errno.ENOSPC, ((2 * 1024) - (1 << 30)))) - sys_fallocate_mock.assert_not_called() - - def test_enough_space_small_blocks(self, sys_fallocate_mock, - fstatvfs_mock): - # Want 1024 bytes in reserve plus 1023 allocated, and have 4 blocks - # of size 512 free, so succeed - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1024') - - fstatvfs_mock.return_value = self._statvfs_result(512, 4) - libc.fallocate(88, 1023) - - def test_not_enough_space_small_blocks(self, sys_fallocate_mock, - fstatvfs_mock): - # Want 1024 bytes in reserve plus 1024 allocated, and have 4 blocks - # of size 512 free, so fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1024') - - fstatvfs_mock.return_value = self._statvfs_result(512, 4) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 1024) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail 1024 <= 1024' - % errno.ENOSPC) - sys_fallocate_mock.assert_not_called() - - def test_free_space_under_reserve(self, sys_fallocate_mock, fstatvfs_mock): - # Want 2048 bytes in reserve but have only 3 blocks of size 512, so - # allocating even 0 bytes fails - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('2048') - - fstatvfs_mock.return_value = self._statvfs_result(512, 3) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 0) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail 1536 <= 2048' - % errno.ENOSPC) - sys_fallocate_mock.assert_not_called() - - def test_all_reserved(self, sys_fallocate_mock, fstatvfs_mock): - # Filesystem is empty, but our reserve is bigger than the - # filesystem, so any allocation will fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('9999999999999') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 100) - self.assertRaises(OSError, libc.fallocate, 88, 0) - sys_fallocate_mock.assert_not_called() - - def test_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock): - # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free - # and file size is 2047, so succeed - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1%') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 3) - libc.fallocate(88, 2047) - - def test_not_enough_space_pct(self, sys_fallocate_mock, fstatvfs_mock): - # Want 1% reserved, filesystem has 3/100 blocks of size 1024 free - # and file size is 2048, so fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('1%') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 3) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 2048) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail 1 <= 1' - % errno.ENOSPC) - sys_fallocate_mock.assert_not_called() - - def test_all_space_reserved_pct(self, sys_fallocate_mock, fstatvfs_mock): - # Filesystem is empty, but our reserve is the whole filesystem, so - # any allocation will fail - libc.FALLOCATE_RESERVE, libc.FALLOCATE_IS_PERCENT = \ - libc.config_fallocate_value('100%') - - fstatvfs_mock.return_value = self._statvfs_result(1024, 100) - with self.assertRaises(OSError) as catcher: - libc.fallocate(88, 0) - self.assertEqual( - str(catcher.exception), - '[Errno %d] FALLOCATE_RESERVE fail 100 <= 100' - % errno.ENOSPC) - sys_fallocate_mock.assert_not_called() - - -@mock.patch('ctypes.get_errno') -@mock.patch.object(libc, '_sys_fallocate') -class TestPunchHole(unittest.TestCase): - def test_punch_hole(self, sys_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = 0 - - libc.punch_hole(123, 456, 789) - - calls = sys_fallocate_mock.mock_calls - self.assertEqual(len(calls), 1) - args = calls[0][1] - self.assertEqual(len(args), 4) - self.assertEqual(args[0], 123) - self.assertEqual( - args[1], libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE) - self.assertEqual(args[2].value, 456) - self.assertEqual(args[3].value, 789) - - def test_error(self, sys_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = -1 - get_errno_mock.return_value = errno.EISDIR - - with self.assertRaises(OSError) as cm: - libc.punch_hole(123, 456, 789) - self.assertEqual(cm.exception.errno, errno.EISDIR) - - def test_arg_bounds(self, sys_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = True - sys_fallocate_mock.return_value = 0 - - with self.assertRaises(ValueError): - libc.punch_hole(0, 1, -1) - with self.assertRaises(ValueError): - libc.punch_hole(0, 1 << 64, 1) - with self.assertRaises(ValueError): - libc.punch_hole(0, -1, 1) - with self.assertRaises(ValueError): - libc.punch_hole(0, 1, 0) - with self.assertRaises(ValueError): - libc.punch_hole(0, 1, 1 << 64) - self.assertEqual([], sys_fallocate_mock.mock_calls) - - # sanity check - libc.punch_hole(0, 0, 1) - self.assertEqual( - [mock.call( - 0, libc.FALLOC_FL_PUNCH_HOLE | libc.FALLOC_FL_KEEP_SIZE, - mock.ANY, mock.ANY)], - sys_fallocate_mock.mock_calls) - # Go confirm the ctypes values separately; apparently == doesn't - # work the way you'd expect with ctypes :-/ - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][2].value, 0) - self.assertEqual(sys_fallocate_mock.mock_calls[0][1][3].value, 1) - - def test_no_fallocate(self, sys_fallocate_mock, get_errno_mock): - sys_fallocate_mock.available = False - - with self.assertRaises(OSError) as cm: - libc.punch_hole(123, 456, 789) - self.assertEqual(cm.exception.errno, errno.ENOTSUP) - - -class TestPunchHoleReally(unittest.TestCase): - def setUp(self): - if not libc._sys_fallocate.available: - raise unittest.SkipTest("libc._sys_fallocate not available") - - def test_punch_a_hole(self): - with tempfile.TemporaryFile() as tf: - tf.write(b"x" * 64 + b"y" * 64 + b"z" * 64) - tf.flush() - - # knock out the first half of the "y"s - libc.punch_hole(tf.fileno(), 64, 32) - - tf.seek(0) - contents = tf.read(4096) - self.assertEqual( - contents, - b"x" * 64 + b"\0" * 32 + b"y" * 32 + b"z" * 64) - - class Test_LibcWrapper(unittest.TestCase): def test_available_function(self): # This should pretty much always exist diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 9f251f2414..851897762e 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -50,7 +50,6 @@ from test.unit import (mock as unit_mock, temptree, mock_check_drive, encode_frag_archive_bodies, skip_if_no_xattrs) from swift.obj import diskfile from swift.common import utils -from swift.common.utils import libc from swift.common.utils import hash_path, mkdirs, Timestamp, \ encode_timestamps, O_TMPFILE, md5 as _md5 from swift.common import ring @@ -4989,7 +4988,7 @@ class DiskFileMixin(BaseDiskFileTestMixin): # This is a horrible hack so you can run this test in isolation. # Some of the ctypes machinery calls os.close(), and that runs afoul # of our mock. - with mock.patch.object(libc, '_sys_fallocate', None): + with mock.patch.object(utils, '_sys_fallocate', None): utils.disable_fallocate() df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc',