openstack/swift
Code Issues Proposed changes

Add support for policy types, 'erasure_coding' policy

Browse Source 
This patch extends the StoragePolicy class for non-replication storage
policies, the first one being "erasure coding".

Changes:

 - Add 'policy_type' support to BaseStoragePolicy class
 - Disallow direct instantiation of BaseStoragePolicy class

 - Subclass BaseStoragePolicy

   - "StoragePolicy":
     . Replication policy, default
     . policy_type = 'replication'

   - "ECStoragePolicy":
     . Erasure Coding policy
     . policy_type = 'erasure_coding'
     . Private member variables
       ec_type (EC backend),
       ec_num_data_fragments (number of fragments original
         data split into after erasure coding operation),
       ec_num_parity_fragments (number of parity fragments
       generated during erasure coding)
     . Private methods
       EC specific attributes and ring validator methods.

 - Swift will use PyECLib, a Python Erasure Coding library, for
   erasure coding operations. PyECLib is already an approved
   OpenStack core requirement.
   (https://bitbucket.org/kmgreen2/pyeclib/)

 - Add test cases for
   - 'policy_type' StoragePolicy member
   - policy_type == 'erasure_coding'

DocImpact

Co-Authored-By: Alistair Coles <alistair.coles@hp.com>
Co-Authored-By: Thiago da Silva <thiago@redhat.com>
Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Co-Authored-By: Paul Luse <paul.e.luse@intel.com>
Co-Authored-By: Samuel Merritt <sam@swiftstack.com>
Co-Authored-By: Christian Schwede <christian.schwede@enovance.com>
Co-Authored-By: Yuan Zhou <yuan.zhou@intel.com>
Change-Id: Ie0e09796e3ec45d3e656fb7540d0e5a5709b8386
Implements: blueprint ec-proxy-work
This commit is contained in:
Tushar Gohad 2014-06-30 11:14:28 -07:00 committed by John Dickinson
parent ce596684f6
commit ed54066288
8 changed files with 778 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
etc/swift.conf-sample
View File

@ -22,9 +22,13 @@ swift_hash_path_prefix = changeme
# defined you must define a policy with index 0 and you must specify a
# default. It is recommended you always define a section for
# storage-policy:0.
#
# A 'policy_type' argument is also supported but is not mandatory. Default
# policy type 'replication' is used when 'policy_type' is unspecified.
[storage-policy:0]
name = Policy-0
default = yes
#policy_type = replication
# the following section would declare a policy called 'silver', the number of
# replicas will be determined by how the ring is built. In this example the
@ -39,6 +43,42 @@ default = yes
# current default.
#[storage-policy:1]
#name = silver
#policy_type = replication
# The following declares a storage policy of type 'erasure_coding' which uses
# Erasure Coding for data reliability. The 'erasure_coding' storage policy in
# Swift is available as a "beta". Please refer to Swift documentation for
# details on how the 'erasure_coding' storage policy is implemented.
#
# Swift uses PyECLib, a Python Erasure coding API library, for encode/decode
# operations. Please refer to Swift documentation for details on how to
# install PyECLib.
#
# When defining an EC policy, 'policy_type' needs to be 'erasure_coding' and
# EC configuration parameters 'ec_type', 'ec_num_data_fragments' and
# 'ec_num_parity_fragments' must be specified. 'ec_type' is chosen from the
# list of EC backends supported by PyECLib. The ring configured for the
# storage policy must have it's "replica" count configured to
# 'ec_num_data_fragments' + 'ec_num_parity_fragments' - this requirement is
# validated when services start. 'ec_object_segment_size' is the amount of
# data that will be buffered up before feeding a segment into the
# encoder/decoder. More information about these configuration options and
# supported `ec_type` schemes is available in the Swift documentation. Please
# refer to Swift documentation for details on how to configure EC policies.
#
# The example 'deepfreeze10-4' policy defined below is a _sample_
# configuration with 10 'data' and 4 'parity' fragments. 'ec_type'
# defines the Erasure Coding scheme. 'jerasure_rs_vand' (Reed-Solomon
# Vandermonde) is used as an example below.
#
#[storage-policy:2]
#name = deepfreeze10-4
#policy_type = erasure_coding
#ec_type = jerasure_rs_vand
#ec_num_data_fragments = 10
#ec_num_parity_fragments = 4
#ec_object_segment_size = 1048576
# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster. These constraints are automatically

379
swift/common/storage_policy.py
View File

@ -17,10 +17,18 @@ import string
from swift.common.utils import config_true_value, SWIFT_CONF_FILE
from swift.common.ring import Ring
from swift.common.utils import quorum_size
from swift.common.exceptions import RingValidationError
from pyeclib.ec_iface import ECDriver, ECDriverError, VALID_EC_TYPES
LEGACY_POLICY_NAME = 'Policy-0'
VALID_CHARS = '-' + string.letters + string.digits
DEFAULT_POLICY_TYPE = REPL_POLICY = 'replication'
EC_POLICY = 'erasure_coding'
DEFAULT_EC_OBJECT_SEGMENT_SIZE = 1048576
class PolicyError(ValueError):
@ -38,36 +46,73 @@ def _get_policy_string(base, policy_index):
return return_string
def get_policy_string(base, policy_index):
def get_policy_string(base, policy_or_index):
"""
Helper function to construct a string from a base and the policy
index. Used to encode the policy index into either a file name
or a directory name by various modules.
Helper function to construct a string from a base and the policy.
Used to encode the policy index into either a file name or a
directory name by various modules.
:param base: the base string
:param policy_index: the storage policy index
:param policy_or_index: StoragePolicy instance, or an index
(string or int), if None the legacy
storage Policy-0 is assumed.
:returns: base name with policy index added
:raises: PolicyError if no policy exists with the given policy_index
"""
if POLICIES.get_by_index(policy_index) is None:
raise PolicyError("No policy with index %r" % policy_index)
return _get_policy_string(base, policy_index)
if isinstance(policy_or_index, BaseStoragePolicy):
policy = policy_or_index
else:
policy = POLICIES.get_by_index(policy_or_index)
if policy is None:
raise PolicyError("Unknown policy", index=policy_or_index)
return _get_policy_string(base, int(policy))
class StoragePolicy(object):
def split_policy_string(policy_string):
"""
Represents a storage policy.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
Helper function to convert a string representing a base and a
policy. Used to decode the policy from either a file name or
a directory name by various modules.
:param policy_string: base name with policy index added
:raises: PolicyError if given index does not map to a valid policy
:returns: a tuple, in the form (base, policy) where base is the base
string and policy is the StoragePolicy instance for the
index encoded in the policy_string.
"""
if '-' in policy_string:
base, policy_index = policy_string.rsplit('-', 1)
else:
base, policy_index = policy_string, None
policy = POLICIES.get_by_index(policy_index)
if get_policy_string(base, policy) != policy_string:
raise PolicyError("Unknown policy", index=policy_index)
return base, policy
class BaseStoragePolicy(object):
"""
Represents a storage policy. Not meant to be instantiated directly;
implement a derived subclasses (e.g. StoragePolicy, ECStoragePolicy, etc)
or use :func:`~swift.common.storage_policy.reload_storage_policies` to
load POLICIES from ``swift.conf``.
The object_ring property is lazy loaded once the service's ``swift_dir``
is known via :meth:`~StoragePolicyCollection.get_object_ring`, but it may
be over-ridden via object_ring kwarg at create time for testing or
actively loaded with :meth:`~StoragePolicy.load_ring`.
"""
policy_type_to_policy_cls = {}
def __init__(self, idx, name='', is_default=False, is_deprecated=False,
object_ring=None):
# do not allow BaseStoragePolicy class to be instantiated directly
if type(self) == BaseStoragePolicy:
raise TypeError("Can't instantiate BaseStoragePolicy directly")
# policy parameter validation
try:
self.idx = int(idx)
except ValueError:
@ -88,6 +133,8 @@ class StoragePolicy(object):
self.name = name
self.is_deprecated = config_true_value(is_deprecated)
self.is_default = config_true_value(is_default)
if self.policy_type not in BaseStoragePolicy.policy_type_to_policy_cls:
raise PolicyError('Invalid type', self.policy_type)
if self.is_deprecated and self.is_default:
raise PolicyError('Deprecated policy can not be default. '
'Invalid config', self.idx)
@ -101,8 +148,80 @@ class StoragePolicy(object):
return cmp(self.idx, int(other))
def __repr__(self):
return ("StoragePolicy(%d, %r, is_default=%s, is_deprecated=%s)") % (
self.idx, self.name, self.is_default, self.is_deprecated)
return ("%s(%d, %r, is_default=%s, "
"is_deprecated=%s, policy_type=%r)") % \
(self.__class__.__name__, self.idx, self.name,
self.is_default, self.is_deprecated, self.policy_type)
@classmethod
def register(cls, policy_type):
"""
Decorator for Storage Policy implementations to register
their StoragePolicy class. This will also set the policy_type
attribute on the registered implementation.
"""
def register_wrapper(policy_cls):
if policy_type in cls.policy_type_to_policy_cls:
raise PolicyError(
'%r is already registered for the policy_type %r' % (
cls.policy_type_to_policy_cls[policy_type],
policy_type))
cls.policy_type_to_policy_cls[policy_type] = policy_cls
policy_cls.policy_type = policy_type
return policy_cls
return register_wrapper
@classmethod
def _config_options_map(cls):
"""
Map config option name to StoragePolicy parameter name.
"""
return {
'name': 'name',
'policy_type': 'policy_type',
'default': 'is_default',
'deprecated': 'is_deprecated',
}
@classmethod
def from_config(cls, policy_index, options):
config_to_policy_option_map = cls._config_options_map()
policy_options = {}
for config_option, value in options.items():
try:
policy_option = config_to_policy_option_map[config_option]
except KeyError:
raise PolicyError('Invalid option %r in '
'storage-policy section' % config_option,
index=policy_index)
policy_options[policy_option] = value
return cls(policy_index, **policy_options)
def get_info(self, config=False):
"""
Return the info dict and conf file options for this policy.
:param config: boolean, if True all config options are returned
"""
info = {}
for config_option, policy_attribute in \
self._config_options_map().items():
info[config_option] = getattr(self, policy_attribute)
if not config:
# remove some options for public consumption
if not self.is_default:
info.pop('default')
if not self.is_deprecated:
info.pop('deprecated')
info.pop('policy_type')
return info
def _validate_ring(self):
"""
Hook, called when the ring is loaded. Can be used to
validate the ring against the StoragePolicy configuration.
"""
pass
def load_ring(self, swift_dir):
"""
@ -114,11 +233,194 @@ class StoragePolicy(object):
return
self.object_ring = Ring(swift_dir, ring_name=self.ring_name)
def get_options(self):
"""Return the valid conf file options for this policy."""
return {'name': self.name,
'default': self.is_default,
'deprecated': self.is_deprecated}
# Validate ring to make sure it conforms to policy requirements
self._validate_ring()
@property
def quorum(self):
"""
Number of successful backend requests needed for the proxy to
consider the client request successful.
"""
raise NotImplementedError()
@BaseStoragePolicy.register(REPL_POLICY)
class StoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'replication'. Default storage policy
class unless otherwise overridden from swift.conf.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
@property
def quorum(self):
"""
Quorum concept in the replication case:
floor(number of replica / 2) + 1
"""
if not self.object_ring:
raise PolicyError('Ring is not loaded')
return quorum_size(self.object_ring.replica_count)
@BaseStoragePolicy.register(EC_POLICY)
class ECStoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'erasure_coding'.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
def __init__(self, idx, name='', is_default=False,
is_deprecated=False, object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None):
super(ECStoragePolicy, self).__init__(
idx, name, is_default, is_deprecated, object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
@property
def ec_type(self):
return self._ec_type
@property
def ec_ndata(self):
return self._ec_ndata
@property
def ec_nparity(self):
return self._ec_nparity
@property
def ec_segment_size(self):
return self._ec_segment_size
def __repr__(self):
return ("%s, EC config(ec_type=%s, ec_segment_size=%d, "
"ec_ndata=%d, ec_nparity=%d)") % (
super(ECStoragePolicy, self).__repr__(), self.ec_type,
self.ec_segment_size, self.ec_ndata, self.ec_nparity)
@classmethod
def _config_options_map(cls):
options = super(ECStoragePolicy, cls)._config_options_map()
options.update({
'ec_type': 'ec_type',
'ec_object_segment_size': 'ec_segment_size',
'ec_num_data_fragments': 'ec_ndata',
'ec_num_parity_fragments': 'ec_nparity',
})
return options
def get_info(self, config=False):
info = super(ECStoragePolicy, self).get_info(config=config)
if not config:
info.pop('ec_object_segment_size')
info.pop('ec_num_data_fragments')
info.pop('ec_num_parity_fragments')
info.pop('ec_type')
return info
def _validate_ring(self):
"""
EC specific validation
Replica count check - we need _at_least_ (#data + #parity) replicas
configured. Also if the replica count is larger than exactly that
number there's a non-zero risk of error for code that is considering
the number of nodes in the primary list from the ring.
"""
if not self.object_ring:
raise PolicyError('Ring is not loaded')
nodes_configured = self.object_ring.replica_count
if nodes_configured != (self.ec_ndata + self.ec_nparity):
raise RingValidationError(
'EC ring for policy %s needs to be configured with '
'exactly %d nodes. Got %d.' % (self.name,
self.ec_ndata + self.ec_nparity, nodes_configured))
@property
def quorum(self):
"""
Number of successful backend requests needed for the proxy to consider
the client request successful.
The quorum size for EC policies defines the minimum number
of data + parity elements required to be able to guarantee
the desired fault tolerance, which is the number of data
elements supplemented by the minimum number of parity
elements required by the chosen erasure coding scheme.
For example, for Reed-Solomon, the minimum number parity
elements required is 1, and thus the quorum_size requirement
is ec_ndata + 1.
Given the number of parity elements required is not the same
for every erasure coding scheme, consult PyECLib for
min_parity_fragments_needed()
"""
return self._ec_quorum_size
class StoragePolicyCollection(object):
@ -236,9 +538,19 @@ class StoragePolicyCollection(object):
:returns: storage policy, or None if no such policy
"""
# makes it easier for callers to just pass in a header value
index = int(index) if index else 0
if index in ('', None):
index = 0
else:
try:
index = int(index)
except ValueError:
return None
return self.by_index.get(index)
@property
def legacy(self):
return self.get_by_index(None)
def get_object_ring(self, policy_idx, swift_dir):
"""
Get the ring object to use to handle a request based on its policy.
@ -267,10 +579,7 @@ class StoragePolicyCollection(object):
# delete from /info if deprecated
if pol.is_deprecated:
continue
policy_entry = {}
policy_entry['name'] = pol.name
if pol.is_default:
policy_entry['default'] = pol.is_default
policy_entry = pol.get_info()
policy_info.append(policy_entry)
return policy_info
@ -287,22 +596,10 @@ def parse_storage_policies(conf):
if not section.startswith('storage-policy:'):
continue
policy_index = section.split(':', 1)[1]
# map config option name to StoragePolicy parameter name
config_to_policy_option_map = {
'name': 'name',
'default': 'is_default',
'deprecated': 'is_deprecated',
}
policy_options = {}
for config_option, value in conf.items(section):
try:
policy_option = config_to_policy_option_map[config_option]
except KeyError:
raise PolicyError('Invalid option %r in '
'storage-policy section %r' % (
config_option, section))
policy_options[policy_option] = value
policy = StoragePolicy(policy_index, **policy_options)
config_options = dict(conf.items(section))
policy_type = config_options.pop('policy_type', DEFAULT_POLICY_TYPE)
policy_cls = BaseStoragePolicy.policy_type_to_policy_cls[policy_type]
policy = policy_cls.from_config(policy_index, config_options)
policies.append(policy)
return StoragePolicyCollection(policies)

13
swift/obj/diskfile.py
View File

@ -63,7 +63,7 @@ from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \
ReplicationLockTimeout, DiskFileExpired, DiskFileXattrNotSupported
from swift.common.swob import multi_range_iterator
from swift.common.storage_policy import get_policy_string, POLICIES
from swift.common.storage_policy import get_policy_string, split_policy_string
from functools import partial
@ -178,11 +178,7 @@ def extract_policy_index(obj_path):
obj_dirname = obj_portion[:obj_portion.index('/')]
except Exception:
return policy_idx
if '-' in obj_dirname:
base, policy_idx = obj_dirname.split('-', 1)
if POLICIES.get_by_index(policy_idx) is None:
policy_idx = 0
return int(policy_idx)
return int(split_policy_string(obj_dirname)[1])
def quarantine_renamer(device_path, corrupted_file_path):
@ -474,11 +470,8 @@ def object_audit_location_generator(devices, mount_check=True, logger=None,
if dir.startswith(DATADIR_BASE)]:
datadir_path = os.path.join(devices, device, dir)
# warn if the object dir doesn't match with a policy
policy_idx = 0
if '-' in dir:
base, policy_idx = dir.split('-', 1)
try:
get_data_dir(policy_idx)
base, policy = split_policy_string(dir)
except ValueError:
if logger:
logger.warn(_('Directory %s does not map to a '

2
test/functional/__init__.py
View File

@ -223,7 +223,7 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
# make policy_to_test be policy index 0 and default for the test config
sp_zero_section = sp_prefix + '0'
conf.add_section(sp_zero_section)
for (k, v) in policy_to_test.get_options().items():
for (k, v) in policy_to_test.get_info(config=True).items():
conf.set(sp_zero_section, k, v)
conf.set(sp_zero_section, 'default', True)

3
test/unit/common/test_internal_client.py
View File

@ -235,7 +235,8 @@ class TestInternalClient(unittest.TestCase):
write_fake_ring(object_ring_path)
with patch_policies([StoragePolicy(0, 'legacy', True)]):
client = internal_client.InternalClient(conf_path, 'test', 1)
self.assertEqual(client.account_ring, client.app.app.app.account_ring)
self.assertEqual(client.account_ring,
client.app.app.app.account_ring)
self.assertEqual(client.account_ring.serialized_path,
account_ring_path)
self.assertEqual(client.container_ring,

410
test/unit/common/test_storage_policy.py
View File

@ -19,8 +19,23 @@ import mock
from tempfile import NamedTemporaryFile
from test.unit import patch_policies, FakeRing
from swift.common.storage_policy import (
StoragePolicy, StoragePolicyCollection, POLICIES, PolicyError,
parse_storage_policies, reload_storage_policies, get_policy_string)
StoragePolicyCollection, POLICIES, PolicyError, parse_storage_policies,
reload_storage_policies, get_policy_string, split_policy_string,
BaseStoragePolicy, StoragePolicy, ECStoragePolicy, REPL_POLICY, EC_POLICY,
VALID_EC_TYPES, DEFAULT_EC_OBJECT_SEGMENT_SIZE)
from swift.common.exceptions import RingValidationError
@BaseStoragePolicy.register('fake')
class FakeStoragePolicy(BaseStoragePolicy):
"""
Test StoragePolicy class - the only user at the moment is
test_validate_policies_type_invalid()
"""
def __init__(self, idx, name='', is_default=False, is_deprecated=False,
object_ring=None):
super(FakeStoragePolicy, self).__init__(
idx, name, is_default, is_deprecated, object_ring)
class TestStoragePolicies(unittest.TestCase):
@ -31,15 +46,35 @@ class TestStoragePolicies(unittest.TestCase):
conf.readfp(StringIO.StringIO(conf_str))
return conf
@patch_policies([StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'one', False),
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False, is_deprecated=True)])
def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs):
try:
f(*args, **kwargs)
except exc_class as err:
err_msg = str(err)
self.assert_(message in err_msg, 'Error message %r did not '
'have expected substring %r' % (err_msg, message))
else:
self.fail('%r did not raise %s' % (message, exc_class.__name__))
def test_policy_baseclass_instantiate(self):
self.assertRaisesWithMessage(TypeError,
"Can't instantiate BaseStoragePolicy",
BaseStoragePolicy, 1, 'one')
@patch_policies([
StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one'),
StoragePolicy(2, 'two'),
StoragePolicy(3, 'three', is_deprecated=True),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=4),
])
def test_swift_info(self):
# the deprecated 'three' should not exist in expect
expect = [{'default': True, 'name': 'zero'},
{'name': 'two'},
{'name': 'one'}]
{'name': 'one'},
{'name': 'ten'}]
swift_info = POLICIES.get_policy_info()
self.assertEquals(sorted(expect, key=lambda k: k['name']),
sorted(swift_info, key=lambda k: k['name']))
@ -48,10 +83,48 @@ class TestStoragePolicies(unittest.TestCase):
def test_get_policy_string(self):
self.assertEquals(get_policy_string('something', 0), 'something')
self.assertEquals(get_policy_string('something', None), 'something')
self.assertEquals(get_policy_string('something', ''), 'something')
self.assertEquals(get_policy_string('something', 1),
'something' + '-1')
self.assertRaises(PolicyError, get_policy_string, 'something', 99)
@patch_policies
def test_split_policy_string(self):
expectations = {
'something': ('something', POLICIES[0]),
'something-1': ('something', POLICIES[1]),
'tmp': ('tmp', POLICIES[0]),
'objects': ('objects', POLICIES[0]),
'tmp-1': ('tmp', POLICIES[1]),
'objects-1': ('objects', POLICIES[1]),
'objects-': PolicyError,
'objects-0': PolicyError,
'objects--1': ('objects-', POLICIES[1]),
'objects-+1': PolicyError,
'objects--': PolicyError,
'objects-foo': PolicyError,
'objects--bar': PolicyError,
'objects-+bar': PolicyError,
# questionable, demonstrated as inverse of get_policy_string
'objects+0': ('objects+0', POLICIES[0]),
'': ('', POLICIES[0]),
'0': ('0', POLICIES[0]),
'-1': ('', POLICIES[1]),
}
for policy_string, expected in expectations.items():
if expected == PolicyError:
try:
invalid = split_policy_string(policy_string)
except PolicyError:
continue # good
else:
self.fail('The string %r returned %r '
'instead of raising a PolicyError' %
(policy_string, invalid))
self.assertEqual(expected, split_policy_string(policy_string))
# should be inverse of get_policy_string
self.assertEqual(policy_string, get_policy_string(*expected))
def test_defaults(self):
self.assertTrue(len(POLICIES) > 0)
@ -66,7 +139,9 @@ class TestStoragePolicies(unittest.TestCase):
def test_storage_policy_repr(self):
test_policies = [StoragePolicy(0, 'aay', True),
StoragePolicy(1, 'bee', False),
StoragePolicy(2, 'cee', False)]
StoragePolicy(2, 'cee', False),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=3)]
policies = StoragePolicyCollection(test_policies)
for policy in policies:
policy_repr = repr(policy)
@ -75,6 +150,13 @@ class TestStoragePolicies(unittest.TestCase):
self.assert_('is_deprecated=%s' % policy.is_deprecated in
policy_repr)
self.assert_(policy.name in policy_repr)
if policy.policy_type == EC_POLICY:
self.assert_('ec_type=%s' % policy.ec_type in policy_repr)
self.assert_('ec_ndata=%s' % policy.ec_ndata in policy_repr)
self.assert_('ec_nparity=%s' %
policy.ec_nparity in policy_repr)
self.assert_('ec_segment_size=%s' %
policy.ec_segment_size in policy_repr)
collection_repr = repr(policies)
collection_repr_lines = collection_repr.splitlines()
self.assert_(policies.__class__.__name__ in collection_repr_lines[0])
@ -157,15 +239,16 @@ class TestStoragePolicies(unittest.TestCase):
def test_validate_policy_params(self):
StoragePolicy(0, 'name') # sanity
# bogus indexes
self.assertRaises(PolicyError, StoragePolicy, 'x', 'name')
self.assertRaises(PolicyError, StoragePolicy, -1, 'name')
self.assertRaises(PolicyError, FakeStoragePolicy, 'x', 'name')
self.assertRaises(PolicyError, FakeStoragePolicy, -1, 'name')
# non-zero Policy-0
self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy,
1, 'policy-0')
self.assertRaisesWithMessage(PolicyError, 'reserved',
FakeStoragePolicy, 1, 'policy-0')
# deprecate default
self.assertRaisesWithMessage(
PolicyError, 'Deprecated policy can not be default',
StoragePolicy, 1, 'Policy-1', is_default=True,
FakeStoragePolicy, 1, 'Policy-1', is_default=True,
is_deprecated=True)
# weird names
names = (
@ -178,7 +261,7 @@ class TestStoragePolicies(unittest.TestCase):
)
for name in names:
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
StoragePolicy, 1, name)
FakeStoragePolicy, 1, name)
def test_validate_policies_names(self):
# duplicate names
@ -188,6 +271,40 @@ class TestStoragePolicies(unittest.TestCase):
self.assertRaises(PolicyError, StoragePolicyCollection,
test_policies)
def test_validate_policies_type_default(self):
# no type specified - make sure the policy is initialized to
# DEFAULT_POLICY_TYPE
test_policy = FakeStoragePolicy(0, 'zero', True)
self.assertEquals(test_policy.policy_type, 'fake')
def test_validate_policies_type_invalid(self):
class BogusStoragePolicy(FakeStoragePolicy):
policy_type = 'bogus'
# unsupported policy type - initialization with FakeStoragePolicy
self.assertRaisesWithMessage(PolicyError, 'Invalid type',
BogusStoragePolicy, 1, 'one')
def test_policies_type_attribute(self):
test_policies = [
StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one'),
StoragePolicy(2, 'two'),
StoragePolicy(3, 'three', is_deprecated=True),
ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=3),
]
policies = StoragePolicyCollection(test_policies)
self.assertEquals(policies.get_by_index(0).policy_type,
REPL_POLICY)
self.assertEquals(policies.get_by_index(1).policy_type,
REPL_POLICY)
self.assertEquals(policies.get_by_index(2).policy_type,
REPL_POLICY)
self.assertEquals(policies.get_by_index(3).policy_type,
REPL_POLICY)
self.assertEquals(policies.get_by_index(10).policy_type,
EC_POLICY)
def test_names_are_normalized(self):
test_policies = [StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'ZERO', False)]
@ -207,16 +324,6 @@ class TestStoragePolicies(unittest.TestCase):
self.assertEqual(pol1, policies.get_by_name(name))
self.assertEqual(policies.get_by_name(name).name, 'One')
def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs):
try:
f(*args, **kwargs)
except exc_class as err:
err_msg = str(err)
self.assert_(message in err_msg, 'Error message %r did not '
'have expected substring %r' % (err_msg, message))
else:
self.fail('%r did not raise %s' % (message, exc_class.__name__))
def test_deprecated_default(self):
bad_conf = self._conf("""
[storage-policy:1]
@ -395,6 +502,133 @@ class TestStoragePolicies(unittest.TestCase):
self.assertRaisesWithMessage(PolicyError, 'Invalid name',
parse_storage_policies, bad_conf)
# policy_type = erasure_coding
# missing ec_type, ec_num_data_fragments and ec_num_parity_fragments
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
""")
self.assertRaisesWithMessage(PolicyError, 'Missing ec_type',
parse_storage_policies, bad_conf)
# missing ec_type, but other options valid...
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_num_data_fragments = 10
ec_num_parity_fragments = 4
""")
self.assertRaisesWithMessage(PolicyError, 'Missing ec_type',
parse_storage_policies, bad_conf)
# ec_type specified, but invalid...
bad_conf = self._conf("""
[storage-policy:0]
name = zero
default = yes
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_type = garbage_alg
ec_num_data_fragments = 10
ec_num_parity_fragments = 4
""")
self.assertRaisesWithMessage(PolicyError,
'Wrong ec_type garbage_alg for policy '
'ec10-4, should be one of "%s"' %
(', '.join(VALID_EC_TYPES)),
parse_storage_policies, bad_conf)
# missing and invalid ec_num_parity_fragments
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_type = jerasure_rs_vand
ec_num_data_fragments = 10
""")
self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_parity_fragments',
parse_storage_policies, bad_conf)
for num_parity in ('-4', '0', 'x'):
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_type = jerasure_rs_vand
ec_num_data_fragments = 10
ec_num_parity_fragments = %s
""" % num_parity)
self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_parity_fragments',
parse_storage_policies, bad_conf)
# missing and invalid ec_num_data_fragments
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_type = jerasure_rs_vand
ec_num_parity_fragments = 4
""")
self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_data_fragments',
parse_storage_policies, bad_conf)
for num_data in ('-10', '0', 'x'):
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_type = jerasure_rs_vand
ec_num_data_fragments = %s
ec_num_parity_fragments = 4
""" % num_data)
self.assertRaisesWithMessage(PolicyError,
'Invalid ec_num_data_fragments',
parse_storage_policies, bad_conf)
# invalid ec_object_segment_size
for segment_size in ('-4', '0', 'x'):
bad_conf = self._conf("""
[storage-policy:0]
name = zero
[storage-policy:1]
name = ec10-4
policy_type = erasure_coding
ec_object_segment_size = %s
ec_type = jerasure_rs_vand
ec_num_data_fragments = 10
ec_num_parity_fragments = 4
""" % segment_size)
self.assertRaisesWithMessage(PolicyError,
'Invalid ec_object_segment_size',
parse_storage_policies, bad_conf)
# Additional section added to ensure parser ignores other sections
conf = self._conf("""
[some-other-section]
@ -430,6 +664,8 @@ class TestStoragePolicies(unittest.TestCase):
self.assertEquals("zero", policies.get_by_index(None).name)
self.assertEquals("zero", policies.get_by_index('').name)
self.assertEqual(policies.get_by_index(0), policies.legacy)
def test_reload_invalid_storage_policies(self):
conf = self._conf("""
[storage-policy:0]
@ -512,18 +748,124 @@ class TestStoragePolicies(unittest.TestCase):
for policy in POLICIES:
self.assertEqual(POLICIES[int(policy)], policy)
def test_storage_policy_get_options(self):
policy = StoragePolicy(1, 'gold', True, False)
self.assertEqual({'name': 'gold',
'default': True,
'deprecated': False},
policy.get_options())
def test_quorum_size_replication(self):
expected_sizes = {1: 1,
2: 2,
3: 2,
4: 3,
5: 3}
for n, expected in expected_sizes.items():
policy = StoragePolicy(0, 'zero',
object_ring=FakeRing(replicas=n))
self.assertEqual(policy.quorum, expected)
policy = StoragePolicy(1, 'gold', False, True)
self.assertEqual({'name': 'gold',
def test_quorum_size_erasure_coding(self):
test_ec_policies = [
ECStoragePolicy(10, 'ec8-2', ec_type='jerasure_rs_vand',
ec_ndata=8, ec_nparity=2),
ECStoragePolicy(11, 'df10-6', ec_type='flat_xor_hd_4',
ec_ndata=10, ec_nparity=6),
]
for ec_policy in test_ec_policies:
k = ec_policy.ec_ndata
expected_size = \
k + ec_policy.pyeclib_driver.min_parity_fragments_needed()
self.assertEqual(expected_size, ec_policy.quorum)
def test_validate_ring(self):
test_policies = [
ECStoragePolicy(0, 'ec8-2', ec_type='jerasure_rs_vand',
ec_ndata=8, ec_nparity=2,
object_ring=FakeRing(replicas=8),
is_default=True),
ECStoragePolicy(1, 'ec10-4', ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=4,
object_ring=FakeRing(replicas=10)),
ECStoragePolicy(2, 'ec4-2', ec_type='jerasure_rs_vand',
ec_ndata=4, ec_nparity=2,
object_ring=FakeRing(replicas=7)),
]
policies = StoragePolicyCollection(test_policies)
for policy in policies:
msg = 'EC ring for policy %s needs to be configured with ' \
'exactly %d nodes.' % \
(policy.name, policy.ec_ndata + policy.ec_nparity)
self.assertRaisesWithMessage(
RingValidationError, msg,
policy._validate_ring)
def test_storage_policy_get_info(self):
test_policies = [
StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_deprecated=True),
ECStoragePolicy(10, 'ten',
ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=3),
ECStoragePolicy(11, 'done', is_deprecated=True,
ec_type='jerasure_rs_vand',
ec_ndata=10, ec_nparity=3),
]
policies = StoragePolicyCollection(test_policies)
expected = {
# default replication
(0, True): {
'name': 'zero',
'default': True,
'deprecated': False,
'policy_type': REPL_POLICY
},
(0, False): {
'name': 'zero',
'default': True,
},
# deprecated replication
(1, True): {
'name': 'one',
'default': False,
'deprecated': True},
policy.get_options())
'deprecated': True,
'policy_type': REPL_POLICY
},
(1, False): {
'name': 'one',
'deprecated': True,
},
# enabled ec
(10, True): {
'name': 'ten',
'default': False,
'deprecated': False,
'policy_type': EC_POLICY,
'ec_type': 'jerasure_rs_vand',
'ec_num_data_fragments': 10,
'ec_num_parity_fragments': 3,
'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,
},
(10, False): {
'name': 'ten',
},
# deprecated ec
(11, True): {
'name': 'done',
'default': False,
'deprecated': True,
'policy_type': EC_POLICY,
'ec_type': 'jerasure_rs_vand',
'ec_num_data_fragments': 10,
'ec_num_parity_fragments': 3,
'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE,
},
(11, False): {
'name': 'done',
'deprecated': True,
},
}
self.maxDiff = None
for policy in policies:
expected_info = expected[(int(policy), True)]
self.assertEqual(policy.get_info(config=True), expected_info)
expected_info = expected[(int(policy), False)]
self.assertEqual(policy.get_info(config=False), expected_info)
if __name__ == '__main__':

5
test/unit/obj/test_diskfile.py
View File

@ -140,9 +140,10 @@ class TestDiskFileModuleMethods(unittest.TestCase):
pn = '/objects-1/0/606/198452b6ef6247c78606/1401379842.14643.data'
self.assertEqual(diskfile.extract_policy_index(pn), 1)
# bad policy index
# well formatted but, unknown policy index
pn = 'objects-2/0/606/198427efcff042c78606/1401379842.14643.data'
self.assertEqual(diskfile.extract_policy_index(pn), 0)
self.assertRaises(ValueError,
diskfile.extract_policy_index, pn)
bad_path = '/srv/node/sda1/objects-t/1/abc/def/1234.data'
self.assertRaises(ValueError,
diskfile.extract_policy_index, bad_path)

1
test/unit/obj/test_server.py
View File

@ -4510,6 +4510,7 @@ class TestObjectServer(unittest.TestCase):
resp.close()
@patch_policies
class TestZeroCopy(unittest.TestCase):
"""Test the object server's zero-copy functionality"""