From 5ecf828b17cf01291ad5c9fafea3c7ddb621af18 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 23 May 2017 16:08:23 -0700 Subject: [PATCH] Follow-up for per-policy proxy configs * Only use one StringIO in ConfigString * Rename the write_affinity_node_count function to be write_affinity_node_count_fn * Use comprehensions instead of six.moves.filter * Rename OverrideConf to ProxyOverrideOptions * Make ProxyOverrideOptions's __repr__ eval()able * Various conf -> options renames * Stop trying to handle a KeyError that should never come up * Be explicit about how deep we need to copy in proxy/test_server.py * Drop an unused return value * Add a test for a non-"proxy-server" app name * Combine bad-section-name tests * Try to clean up (at least a little) a self-described "hokey test" Related-Change: I3f718f425f525baa80045ba067950c752bcaaefc Change-Id: I4e81175d5445049bc1f48b3ac02c5bc0f77e6f59 --- etc/proxy-server.conf-sample | 12 +- swift/common/wsgi.py | 2 +- swift/proxy/controllers/obj.py | 13 +- swift/proxy/server.py | 92 +++++----- test/unit/proxy/controllers/test_obj.py | 2 +- test/unit/proxy/test_server.py | 229 +++++++++++++----------- 6 files changed, 186 insertions(+), 164 deletions(-) diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index ce4c656caf..e439efe67a 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -167,11 +167,11 @@ use = egg:swift#proxy # Depth of the proxy put queue. # put_queue_depth = 10 # -# Storage nodes can be chosen at random (shuffle), by using timing -# measurements (timing), or by using an explicit match (affinity). -# Using timing measurements may allow for lower overall latency, while -# using affinity allows for finer control. In both the timing and -# affinity cases, equally-sorting nodes are still randomly chosen to +# During GET and HEAD requests, storage nodes can be chosen at random +# (shuffle), by using timing measurements (timing), or by using an explicit +# region/zone match (affinity). Using timing measurements may allow for lower +# overall latency, while using affinity allows for finer control. In both the +# timing and affinity cases, equally-sorting nodes are still randomly chosen to # spread load. # The valid values for sorting_method are "affinity", "shuffle", or "timing". # This option may be overridden in a per-policy configuration section. @@ -215,7 +215,7 @@ use = egg:swift#proxy # This option may be overridden in a per-policy configuration section. # read_affinity = # -# Specifies which backend servers to prefer on writes. Format is a comma +# Specifies which backend servers to prefer on object writes. Format is a comma # separated list of affinity descriptors of the form r for region N or # rz for region N, zone M. If this is set, then when handling an object # PUT request, some number (see setting write_affinity_node_count) of local diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index a6dcd226b8..c3ac7c4f08 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -115,7 +115,7 @@ class ConfigString(NamedConfigLoader): self.filename = "string" defaults = { 'here': "string", - '__file__': StringIO(dedent(config_string)), + '__file__': self.contents, } self.parser = loadwsgi.NicerConfigParser("string", defaults=defaults) self.parser.optionxform = str # Don't lower-case keys diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index e9c9b5e1bb..962a35c9f3 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -24,7 +24,6 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. -import six from six.moves.urllib.parse import unquote import collections @@ -143,26 +142,26 @@ class BaseObjectController(Controller): :param ring: ring to get nodes from :param partition: ring partition to yield nodes for """ - policy_conf = self.app.get_policy_options(policy) - is_local = policy_conf.write_affinity_is_local_fn + policy_options = self.app.get_policy_options(policy) + is_local = policy_options.write_affinity_is_local_fn if is_local is None: return self.app.iter_nodes(ring, partition, policy=policy) primary_nodes = ring.get_part_nodes(partition) - num_locals = policy_conf.write_affinity_node_count(len(primary_nodes)) + num_locals = policy_options.write_affinity_node_count_fn( + len(primary_nodes)) all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition)) first_n_local_nodes = list(itertools.islice( - six.moves.filter(is_local, all_nodes), num_locals)) + (node for node in all_nodes if is_local(node)), num_locals)) # refresh it; it moved when we computed first_n_local_nodes all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition)) local_first_node_iter = itertools.chain( first_n_local_nodes, - six.moves.filter(lambda node: node not in first_n_local_nodes, - all_nodes)) + (node for node in all_nodes if node not in first_n_local_nodes)) return self.app.iter_nodes( ring, partition, node_iter=local_first_node_iter, policy=policy) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 216f8a1840..ba0fc58bea 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -85,20 +85,20 @@ def _label_for_policy(policy): return '(default)' -class OverrideConf(object): +class ProxyOverrideOptions(object): """ - Encapsulates proxy server properties that may be overridden e.g. for + Encapsulates proxy server options that may be overridden e.g. for policy specific configurations. :param conf: the proxy-server config dict. :param override_conf: a dict of overriding configuration options. """ def __init__(self, base_conf, override_conf): - self.conf = base_conf - self.override_conf = override_conf + def get(key, default): + return override_conf.get(key, base_conf.get(key, default)) - self.sorting_method = self._get('sorting_method', 'shuffle').lower() - self.read_affinity = self._get('read_affinity', '') + self.sorting_method = get('sorting_method', 'shuffle').lower() + self.read_affinity = get('read_affinity', '') try: self.read_affinity_sort_key = affinity_key_function( self.read_affinity) @@ -107,7 +107,7 @@ class OverrideConf(object): raise ValueError("Invalid read_affinity value: %r (%s)" % (self.read_affinity, err.message)) - self.write_affinity = self._get('write_affinity', '') + self.write_affinity = get('write_affinity', '') try: self.write_affinity_is_local_fn \ = affinity_locality_predicate(self.write_affinity) @@ -115,15 +115,15 @@ class OverrideConf(object): # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (self.write_affinity, err.message)) - self.write_affinity_node_value = self._get( + self.write_affinity_node_count = get( 'write_affinity_node_count', '2 * replicas').lower() - value = self.write_affinity_node_value.split() + value = self.write_affinity_node_count.split() if len(value) == 1: wanc_value = int(value[0]) - self.write_affinity_node_count = lambda replicas: wanc_value + self.write_affinity_node_count_fn = lambda replicas: wanc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': wanc_value = int(value[0]) - self.write_affinity_node_count = \ + self.write_affinity_node_count_fn = \ lambda replicas: wanc_value * replicas else: raise ValueError( @@ -131,13 +131,21 @@ class OverrideConf(object): (' '.join(value))) def __repr__(self): - return ('sorting_method: %s, read_affinity: %s, write_affinity: %s, ' - 'write_affinity_node_count: %s' % - (self.sorting_method, self.read_affinity, self.write_affinity, - self.write_affinity_node_value)) + return '%s({}, {%s})' % (self.__class__.__name__, ', '.join( + '%r: %r' % (k, getattr(self, k)) for k in ( + 'sorting_method', + 'read_affinity', + 'write_affinity', + 'write_affinity_node_count'))) - def _get(self, key, default): - return self.override_conf.get(key, self.conf.get(key, default)) + def __eq__(self, other): + if not isinstance(other, ProxyOverrideOptions): + return False + return all(getattr(self, k) == getattr(other, k) for k in ( + 'sorting_method', + 'read_affinity', + 'write_affinity', + 'write_affinity_node_count')) class Application(object): @@ -151,9 +159,9 @@ class Application(object): self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger - self._override_confs = self._load_per_policy_config(conf) + self._override_options = self._load_per_policy_config(conf) self.sorts_by_timing = any(pc.sorting_method == 'timing' - for pc in self._override_confs.values()) + for pc in self._override_options.values()) self._error_limiting = {} @@ -277,7 +285,7 @@ class Application(object): def _make_policy_override(self, policy, conf, override_conf): label_for_policy = _label_for_policy(policy) try: - override = OverrideConf(conf, override_conf) + override = ProxyOverrideOptions(conf, override_conf) self.logger.debug("Loaded override config for %s: %r" % (label_for_policy, override)) return override @@ -290,15 +298,16 @@ class Application(object): :param conf: the proxy server local conf dict :return: a dict mapping :class:`BaseStoragePolicy` to an instance of - :class:`OverrideConf` that has policy specific config attributes + :class:`ProxyOverrideOptions` that has policy-specific config + attributes """ - # the default conf will be used when looking up a policy that had no - # override conf - default_conf = self._make_policy_override(None, conf, {}) - override_confs = defaultdict(lambda: default_conf) + # the default options will be used when looking up a policy that had no + # override options + default_options = self._make_policy_override(None, conf, {}) + overrides = defaultdict(lambda: default_options) # force None key to be set in the defaultdict so that it is found when # iterating over items in check_config - override_confs[None] = default_conf + overrides[None] = default_options for index, override_conf in conf.get('policy_config', {}).items(): try: index = int(index) @@ -313,29 +322,29 @@ class Application(object): raise ValueError( "No policy found for override config, index: %s" % index) override = self._make_policy_override(policy, conf, override_conf) - override_confs[policy] = override - return override_confs + overrides[policy] = override + return overrides def get_policy_options(self, policy): """ Return policy specific options. :param policy: an instance of :class:`BaseStoragePolicy` - :return: an instance of :class:`OverrideConf` + :return: an instance of :class:`ProxyOverrideOptions` """ - return self._override_confs[policy] + return self._override_options[policy] def check_config(self): """ Check the configuration for possible errors """ - for policy, conf in self._override_confs.items(): - if conf.read_affinity and conf.sorting_method != 'affinity': + for policy, options in self._override_options.items(): + if options.read_affinity and options.sorting_method != 'affinity': self.logger.warning( _("sorting_method is set to '%(method)s', not 'affinity'; " "%(label)s read_affinity setting will have no effect."), {'label': _label_for_policy(policy), - 'method': conf.sorting_method}) + 'method': options.sorting_method}) def get_object_ring(self, policy_idx): """ @@ -531,16 +540,16 @@ class Application(object): # (ie within the rounding resolution) won't prefer one over another. # Python's sort is stable (http://wiki.python.org/moin/HowTo/Sorting/) shuffle(nodes) - policy_conf = self.get_policy_options(policy) - if policy_conf.sorting_method == 'timing': + policy_options = self.get_policy_options(policy) + if policy_options.sorting_method == 'timing': now = time() def key_func(node): timing, expires = self.node_timings.get(node['ip'], (-1.0, 0)) return timing if expires > now else -1.0 nodes.sort(key=key_func) - elif policy_conf.sorting_method == 'affinity': - nodes.sort(key=policy_conf.read_affinity_sort_key) + elif policy_options.sorting_method == 'affinity': + nodes.sort(key=policy_options.read_affinity_sort_key) return nodes def set_node_timing(self, node, timing): @@ -683,14 +692,7 @@ def parse_per_policy_config(conf): :raises ValueError: if a policy config section has an invalid name """ policy_config = {} - try: - all_conf = readconf(conf['__file__']) - except KeyError: - get_logger(conf).warning( - "Unable to load policy specific configuration options: " - "cannot access proxy server conf file") - return policy_config - + all_conf = readconf(conf['__file__']) policy_section_prefix = conf['__name__'] + ':policy:' for section, options in all_conf.items(): if not section.startswith(policy_section_prefix): diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 119429c0ee..e91c104cd3 100644 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -218,7 +218,7 @@ class BaseObjectControllerMixin(object): policy_conf.write_affinity_is_local_fn = ( lambda node: node['region'] == 1) # we'll write to one more than replica count local nodes - policy_conf.write_affinity_node_count = lambda r: r + 1 + policy_conf.write_affinity_node_count_fn = lambda r: r + 1 object_ring = self.policy.object_ring # make our fake ring have plenty of nodes, and not get limited diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 2f1e96ef5f..07b74a5a25 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -40,7 +40,6 @@ import re import random from collections import defaultdict import uuid -from copy import deepcopy import mock from eventlet import sleep, spawn, wsgi, Timeout, debug @@ -753,9 +752,8 @@ class TestProxyServer(unittest.TestCase): node_timings=None): # Note with shuffling mocked out, sort_nodes will by default return # nodes in the order they are given - nodes = deepcopy(nodes) - conf = deepcopy(conf) - conf['policy_config'] = deepcopy(policy_conf) + nodes = list(nodes) + conf = dict(conf, policy_config=policy_conf) baseapp = proxy_server.Application(conf, FakeMemcache(), logger=FakeLogger(), @@ -1298,17 +1296,17 @@ class TestProxyServerConfigLoading(unittest.TestCase): f.write(dedent(conf_body)) return conf_path - def _write_conf_and_load_app(self, conf_sections): + def _write_conf_and_load_app(self, conf_sections, app_name='proxy-server'): # write proxy-server.conf file, load app - conf_body = """ + conf_body = dedent(""" [DEFAULT] swift_dir = %s [pipeline:main] - pipeline = proxy-server + pipeline = %s %s - """ % (self.tempdir, conf_sections) + """) % (self.tempdir, app_name, dedent(conf_sections)) conf_path = self._write_conf(conf_body) with mock.patch('swift.proxy.server.get_logger', @@ -1316,12 +1314,12 @@ class TestProxyServerConfigLoading(unittest.TestCase): app = loadapp(conf_path, allow_modify_pipeline=False) return app - def _check_policy_conf(self, app, exp_conf, exp_is_local): + def _check_policy_options(self, app, exp_options, exp_is_local): # verify expected config - for policy, options in exp_conf.items(): + for policy, options in exp_options.items(): for k, v in options.items(): actual = getattr(app.get_policy_options(policy), k) - if k == "write_affinity_node_count": + if k == "write_affinity_node_count_fn": if policy: # this check only applies when using a policy actual = actual(policy.object_ring.replica_count) self.assertEqual(v, actual) @@ -1340,7 +1338,6 @@ class TestProxyServerConfigLoading(unittest.TestCase): self.assertIs(expected_result, actual, "Expected %s but got %s for %s, policy %s" % (expected_result, actual, node, policy)) - return app def test_per_policy_conf_none_configured(self): conf_sections = """ @@ -1349,14 +1346,14 @@ class TestProxyServerConfigLoading(unittest.TestCase): """ expected_default = {"read_affinity": "", "sorting_method": "shuffle", - "write_affinity_node_count": 6} - exp_conf = {None: expected_default, - POLICIES[0]: expected_default, - POLICIES[1]: expected_default} + "write_affinity_node_count_fn": 6} + exp_options = {None: expected_default, + POLICIES[0]: expected_default, + POLICIES[1]: expected_default} exp_is_local = {POLICIES[0]: None, POLICIES[1]: None} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, exp_is_local) + self._check_policy_options(app, exp_options, exp_is_local) def test_per_policy_conf_one_configured(self): conf_sections = """ @@ -1371,30 +1368,39 @@ class TestProxyServerConfigLoading(unittest.TestCase): """ expected_default = {"read_affinity": "", "sorting_method": "shuffle", - "write_affinity_node_count": 6} - exp_conf = {None: expected_default, - POLICIES[0]: {"read_affinity": "r1=100", - "sorting_method": "affinity", - "write_affinity_node_count": 3}, - POLICIES[1]: expected_default} + "write_affinity_node_count_fn": 6} + exp_options = {None: expected_default, + POLICIES[0]: {"read_affinity": "r1=100", + "sorting_method": "affinity", + "write_affinity_node_count_fn": 3}, + POLICIES[1]: expected_default} exp_is_local = {POLICIES[0]: [({'region': 1, 'zone': 2}, True), ({'region': 2, 'zone': 1}, False)], POLICIES[1]: None} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, exp_is_local) + self._check_policy_options(app, exp_options, exp_is_local) - default_conf = app.get_policy_options(None) + default_options = app.get_policy_options(None) self.assertEqual( - ('sorting_method: shuffle, read_affinity: , write_affinity: , ' - 'write_affinity_node_count: 2 * replicas'), - repr(default_conf)) - policy_0_conf = app.get_policy_options(POLICIES[0]) + "ProxyOverrideOptions({}, {'sorting_method': 'shuffle', " + "'read_affinity': '', 'write_affinity': '', " + "'write_affinity_node_count': '2 * replicas'})", + repr(default_options)) + self.assertEqual(default_options, eval(repr(default_options), { + 'ProxyOverrideOptions': default_options.__class__})) + + policy_0_options = app.get_policy_options(POLICIES[0]) self.assertEqual( - ('sorting_method: affinity, read_affinity: r1=100, ' - 'write_affinity: r1, write_affinity_node_count: 1 * replicas'), - repr(policy_0_conf)) - policy_1_conf = app.get_policy_options(POLICIES[1]) - self.assertIs(default_conf, policy_1_conf) + "ProxyOverrideOptions({}, {'sorting_method': 'affinity', " + "'read_affinity': 'r1=100', 'write_affinity': 'r1', " + "'write_affinity_node_count': '1 * replicas'})", + repr(policy_0_options)) + self.assertEqual(policy_0_options, eval(repr(policy_0_options), { + 'ProxyOverrideOptions': policy_0_options.__class__})) + self.assertNotEqual(default_options, policy_0_options) + + policy_1_options = app.get_policy_options(POLICIES[1]) + self.assertIs(default_options, policy_1_options) def test_per_policy_conf_inherits_defaults(self): conf_sections = """ @@ -1409,17 +1415,17 @@ class TestProxyServerConfigLoading(unittest.TestCase): """ expected_default = {"read_affinity": "", "sorting_method": "affinity", - "write_affinity_node_count": 3} - exp_conf = {None: expected_default, - POLICIES[0]: {"read_affinity": "r1=100", - "sorting_method": "affinity", - "write_affinity_node_count": 3}, - POLICIES[1]: expected_default} + "write_affinity_node_count_fn": 3} + exp_options = {None: expected_default, + POLICIES[0]: {"read_affinity": "r1=100", + "sorting_method": "affinity", + "write_affinity_node_count_fn": 3}, + POLICIES[1]: expected_default} exp_is_local = {POLICIES[0]: [({'region': 1, 'zone': 2}, True), ({'region': 2, 'zone': 1}, False)], POLICIES[1]: None} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, exp_is_local) + self._check_policy_options(app, exp_options, exp_is_local) def test_per_policy_conf_overrides_default_affinity(self): conf_sections = """ @@ -1440,22 +1446,22 @@ class TestProxyServerConfigLoading(unittest.TestCase): write_affinity = r3 write_affinity_node_count = 4 """ - exp_conf = {None: {"read_affinity": "r2=10", - "sorting_method": "affinity", - "write_affinity_node_count": 3}, - POLICIES[0]: {"read_affinity": "r1=100", - "sorting_method": "affinity", - "write_affinity_node_count": 5}, - POLICIES[1]: {"read_affinity": "r1=1", - "sorting_method": "affinity", - "write_affinity_node_count": 4}} + exp_options = {None: {"read_affinity": "r2=10", + "sorting_method": "affinity", + "write_affinity_node_count_fn": 3}, + POLICIES[0]: {"read_affinity": "r1=100", + "sorting_method": "affinity", + "write_affinity_node_count_fn": 5}, + POLICIES[1]: {"read_affinity": "r1=1", + "sorting_method": "affinity", + "write_affinity_node_count_fn": 4}} exp_is_local = {POLICIES[0]: [({'region': 1, 'zone': 2}, True), ({'region': 2, 'zone': 1}, False)], POLICIES[1]: [({'region': 3, 'zone': 2}, True), ({'region': 1, 'zone': 1}, False), ({'region': 2, 'zone': 1}, False)]} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, exp_is_local) + self._check_policy_options(app, exp_options, exp_is_local) def test_per_policy_conf_overrides_default_sorting_method(self): conf_sections = """ @@ -1471,14 +1477,14 @@ class TestProxyServerConfigLoading(unittest.TestCase): sorting_method = affinity read_affinity = r1=1 """ - exp_conf = {None: {"read_affinity": "", - "sorting_method": "timing"}, - POLICIES[0]: {"read_affinity": "r1=100", - "sorting_method": "affinity"}, - POLICIES[1]: {"read_affinity": "r1=1", - "sorting_method": "affinity"}} + exp_options = {None: {"read_affinity": "", + "sorting_method": "timing"}, + POLICIES[0]: {"read_affinity": "r1=100", + "sorting_method": "affinity"}, + POLICIES[1]: {"read_affinity": "r1=1", + "sorting_method": "affinity"}} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, {}) + self._check_policy_options(app, exp_options, {}) def test_per_policy_conf_with_DEFAULT_options(self): conf_body = """ @@ -1507,25 +1513,27 @@ class TestProxyServerConfigLoading(unittest.TestCase): sorting_method = affinity """ % self.tempdir + # Don't just use _write_conf_and_load_app, as we don't want to have + # duplicate DEFAULT sections conf_path = self._write_conf(conf_body) with mock.patch('swift.proxy.server.get_logger', return_value=FakeLogger()): app = loadapp(conf_path, allow_modify_pipeline=False) - exp_conf = { + exp_options = { # default read_affinity is r1, set in proxy-server section None: {"read_affinity": "r1=100", "sorting_method": "shuffle", - "write_affinity_node_count": 6}, + "write_affinity_node_count_fn": 6}, # policy 0 read affinity is r2, dictated by policy 0 section POLICIES[0]: {"read_affinity": "r2=100", "sorting_method": "affinity", - "write_affinity_node_count": 6}, + "write_affinity_node_count_fn": 6}, # policy 1 read_affinity is r0, dictated by DEFAULT section, # overrides proxy server section POLICIES[1]: {"read_affinity": "r0=100", "sorting_method": "affinity", - "write_affinity_node_count": 6}} + "write_affinity_node_count_fn": 6}} exp_is_local = { # default write_affinity is r0, dictated by DEFAULT section None: [({'region': 0, 'zone': 2}, True), @@ -1536,7 +1544,7 @@ class TestProxyServerConfigLoading(unittest.TestCase): # policy 1 write_affinity is r0, inherited from default POLICIES[1]: [({'region': 0, 'zone': 2}, True), ({'region': 1, 'zone': 1}, False)]} - self._check_policy_conf(app, exp_conf, exp_is_local) + self._check_policy_options(app, exp_options, exp_is_local) def test_per_policy_conf_warns_about_sorting_method_mismatch(self): # verify that policy specific warnings are emitted when read_affinity @@ -1554,14 +1562,14 @@ class TestProxyServerConfigLoading(unittest.TestCase): sorting_method = affinity read_affinity = r1=1 """ - exp_conf = {None: {"read_affinity": "r2=10", - "sorting_method": "timing"}, - POLICIES[0]: {"read_affinity": "r1=100", - "sorting_method": "timing"}, - POLICIES[1]: {"read_affinity": "r1=1", - "sorting_method": "affinity"}} + exp_options = {None: {"read_affinity": "r2=10", + "sorting_method": "timing"}, + POLICIES[0]: {"read_affinity": "r1=100", + "sorting_method": "timing"}, + POLICIES[1]: {"read_affinity": "r1=1", + "sorting_method": "affinity"}} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, {}) + self._check_policy_options(app, exp_options, {}) lines = app.logger.get_lines_for_level('warning') scopes = {'default', 'policy 0 (nulo)'} for line in lines[:2]: @@ -1575,6 +1583,25 @@ class TestProxyServerConfigLoading(unittest.TestCase): self.fail("None of %s found in warning: %r" % (scopes, line)) self.assertFalse(scopes) + def test_per_policy_conf_section_name_inherits_from_app_section_name(self): + conf_sections = """ + [app:proxy-srv] + use = egg:swift#proxy + sorting_method = affinity + + [proxy-server:policy:0] + sorting_method = timing + # ignored! + + [proxy-srv:policy:1] + sorting_method = shuffle + """ + exp_options = {None: {'sorting_method': 'affinity'}, + POLICIES[0]: {'sorting_method': 'affinity'}, + POLICIES[1]: {'sorting_method': 'shuffle'}} + app = self._write_conf_and_load_app(conf_sections, 'proxy-srv') + self._check_policy_options(app, exp_options, {}) + def test_per_policy_conf_with_unknown_policy(self): # verify that unknown policy section is warned about but doesn't break # other policy configs @@ -1604,14 +1631,14 @@ class TestProxyServerConfigLoading(unittest.TestCase): [proxy-server:policy:1] read_affinity = r1=1 """ - exp_conf = {None: {"read_affinity": "", - "sorting_method": "affinity"}, - POLICIES[0]: {"read_affinity": "", - "sorting_method": "timing"}, - POLICIES[1]: {"read_affinity": "r1=1", - "sorting_method": "affinity"}} + exp_options = {None: {"read_affinity": "", + "sorting_method": "affinity"}, + POLICIES[0]: {"read_affinity": "", + "sorting_method": "timing"}, + POLICIES[1]: {"read_affinity": "r1=1", + "sorting_method": "affinity"}} app = self._write_conf_and_load_app(conf_sections) - self._check_policy_conf(app, exp_conf, {}) + self._check_policy_options(app, exp_options, {}) def test_per_policy_conf_invalid_read_affinity_value(self): def do_test(conf_sections, scope): @@ -1707,28 +1734,22 @@ class TestProxyServerConfigLoading(unittest.TestCase): do_test(conf_sections, '(default)') def test_per_policy_conf_bad_section_name(self): - conf_sections = """ - [app:proxy-server] - use = egg:swift#proxy + def do_test(policy): + conf_sections = """ + [app:proxy-server] + use = egg:swift#proxy - [proxy-server:policy:] - """ - with self.assertRaises(ValueError) as cm: - self._write_conf_and_load_app(conf_sections) - self.assertIn("Override config must refer to policy index: ''", - cm.exception.message) + [proxy-server:policy:%s] + """ % policy + with self.assertRaises(ValueError) as cm: + self._write_conf_and_load_app(conf_sections) + self.assertEqual( + "Override config must refer to policy index: %r" % policy, + cm.exception.message) - def test_per_policy_conf_section_name_not_index(self): - conf_sections = """ - [app:proxy-server] - use = egg:swift#proxy - - [proxy-server:policy:uno] - """ - with self.assertRaises(ValueError) as cm: - self._write_conf_and_load_app(conf_sections) - self.assertIn("Override config must refer to policy index: 'uno'", - cm.exception.message) + do_test('') + do_test('uno') + do_test('0.0') class TestProxyServerConfigStringLoading(TestProxyServerConfigLoading): @@ -2635,9 +2656,9 @@ class TestReplicatedObjectController( object_ring = self.app.get_object_ring(0) object_ring.max_more_nodes = 100 - policy_conf = self.app.get_policy_options(POLICIES[0]) - policy_conf.write_affinity_is_local_fn = is_r0 - policy_conf.write_affinity_node_count = lambda r: 3 + policy_options = self.app.get_policy_options(POLICIES[0]) + policy_options.write_affinity_is_local_fn = is_r0 + policy_options.write_affinity_node_count_fn = lambda r: 3 controller = \ ReplicatedObjectController( @@ -2654,13 +2675,13 @@ class TestReplicatedObjectController( res = controller.PUT(req) self.assertTrue(res.status.startswith('201 ')) - self.assertEqual(3, len(written_to)) # this is kind of a hokey test, but in FakeRing, the port is even when # the region is 0, and odd when the region is 1, so this test asserts # that we wrote to 2 nodes in region 0, then went to 1 non-r0 node. - self.assertEqual(0, written_to[0][1] % 2) # it's (ip, port, device) - self.assertEqual(0, written_to[1][1] % 2) - self.assertNotEqual(0, written_to[2][1] % 2) + def get_region(x): + return x[1] % 2 # it's (ip, port, device) + + self.assertEqual([0, 0, 1], [get_region(x) for x in written_to]) @unpatch_policies def test_PUT_no_etag_fallocate(self):