From 32a19fd56c21775f4e53c141aabffb8a7e9883eb Mon Sep 17 00:00:00 2001 From: Maurice Schreiber Date: Tue, 4 Sep 2018 15:24:06 +0200 Subject: [PATCH] speed up GET scheduler-stats/pools/detail return a cached host state map, which gets updated on each scheduling Change-Id: Ia258164a43fc15f83dfadfc1133ab4faa972e41c Closes-Bug: #1804659 --- manila/api/v1/scheduler_stats.py | 3 ++- manila/scheduler/drivers/filter.py | 4 ++-- manila/scheduler/host_manager.py | 5 +++-- manila/scheduler/manager.py | 6 +++--- manila/scheduler/rpcapi.py | 10 ++++++---- manila/tests/api/v1/test_scheduler_stats.py | 15 ++++++++++----- manila/tests/scheduler/test_manager.py | 3 ++- manila/tests/scheduler/test_rpcapi.py | 2 +- ...59-speed-up-pools-detail-18f539a96042099a.yaml | 5 +++++ 9 files changed, 34 insertions(+), 19 deletions(-) create mode 100644 releasenotes/notes/bug-1804659-speed-up-pools-detail-18f539a96042099a.yaml diff --git a/manila/api/v1/scheduler_stats.py b/manila/api/v1/scheduler_stats.py index d7f715ce2f..3067cd9b58 100644 --- a/manila/api/v1/scheduler_stats.py +++ b/manila/api/v1/scheduler_stats.py @@ -71,7 +71,8 @@ class SchedulerStatsController(wsgi.Controller): msg = _("Share type %s not found.") % req_share_type raise exc.HTTPBadRequest(explanation=msg) - pools = self.scheduler_api.get_pools(context, filters=search_opts) + pools = self.scheduler_api.get_pools(context, filters=search_opts, + cached=True) detail = (action == 'detail') return self._view_builder.pools(pools, detail=detail) diff --git a/manila/scheduler/drivers/filter.py b/manila/scheduler/drivers/filter.py index 2f7cd57b00..cea54458b3 100644 --- a/manila/scheduler/drivers/filter.py +++ b/manila/scheduler/drivers/filter.py @@ -45,8 +45,8 @@ class FilterScheduler(base.Scheduler): """Fetch options dictionary. Broken out for testing.""" return self.options.get_configuration() - def get_pools(self, context, filters): - return self.host_manager.get_pools(context, filters) + def get_pools(self, context, filters, cached): + return self.host_manager.get_pools(context, filters, cached) def _post_select_populate_filter_properties(self, filter_properties, host_state): diff --git a/manila/scheduler/host_manager.py b/manila/scheduler/host_manager.py index 1c7344ee39..2c7cacab1d 100644 --- a/manila/scheduler/host_manager.py +++ b/manila/scheduler/host_manager.py @@ -626,9 +626,10 @@ class HostManager(object): return all_pools.values() - def get_pools(self, context, filters=None): + def get_pools(self, context, filters=None, cached=False): """Returns a dict of all pools on all hosts HostManager knows about.""" - self._update_host_state_map(context) + if not cached or not self.host_state_map: + self._update_host_state_map(context) all_pools = [] for host, host_state in self.host_state_map.items(): diff --git a/manila/scheduler/manager.py b/manila/scheduler/manager.py index 525a20957d..3a5957e5a2 100644 --- a/manila/scheduler/manager.py +++ b/manila/scheduler/manager.py @@ -62,7 +62,7 @@ MAPPING = { class SchedulerManager(manager.Manager): """Chooses a host to create shares.""" - RPC_API_VERSION = '1.8' + RPC_API_VERSION = '1.9' def __init__(self, scheduler_driver=None, service_name=None, *args, **kwargs): @@ -120,9 +120,9 @@ class SchedulerManager(manager.Manager): 'create_share', {'status': constants.STATUS_ERROR}, context, ex, request_spec) - def get_pools(self, context, filters=None): + def get_pools(self, context, filters=None, cached=False): """Get active pools from the scheduler's cache.""" - return self.driver.get_pools(context, filters) + return self.driver.get_pools(context, filters, cached) def manage_share(self, context, share_id, driver_options, request_spec, filter_properties=None): diff --git a/manila/scheduler/rpcapi.py b/manila/scheduler/rpcapi.py index 3dcbf48e78..2195420b97 100644 --- a/manila/scheduler/rpcapi.py +++ b/manila/scheduler/rpcapi.py @@ -40,9 +40,10 @@ class SchedulerAPI(object): 1.6 - Add manage_share 1.7 - Updated migrate_share_to_host method with new parameters 1.8 - Rename create_consistency_group -> create_share_group method + 1.9 - Add cached parameter to get_pools method """ - RPC_API_VERSION = '1.8' + RPC_API_VERSION = '1.9' def __init__(self): super(SchedulerAPI, self).__init__() @@ -69,9 +70,10 @@ class SchedulerAPI(object): host=host, capabilities=capabilities) - def get_pools(self, context, filters=None): - call_context = self.client.prepare(version='1.1') - return call_context.call(context, 'get_pools', filters=filters) + def get_pools(self, context, filters=None, cached=False): + call_context = self.client.prepare(version='1.9') + return call_context.call(context, 'get_pools', filters=filters, + cached=cached) def create_share_group(self, context, share_group_id, request_spec=None, filter_properties=None): diff --git a/manila/tests/api/v1/test_scheduler_stats.py b/manila/tests/api/v1/test_scheduler_stats.py index 02c3ce7b9b..1c2b65979c 100644 --- a/manila/tests/api/v1/test_scheduler_stats.py +++ b/manila/tests/api/v1/test_scheduler_stats.py @@ -101,7 +101,8 @@ class SchedulerStatsControllerTestCase(test.TestCase): } self.assertDictMatch(result, expected) - mock_get_pools.assert_called_once_with(self.ctxt, filters={}) + mock_get_pools.assert_called_once_with(self.ctxt, filters={}, + cached=True) self.mock_policy_check.assert_called_once_with( self.ctxt, self.resource_name, 'index') @@ -148,7 +149,8 @@ class SchedulerStatsControllerTestCase(test.TestCase): self.assertDictMatch(result, expected_result) mock_get_pools.assert_called_once_with(self.ctxt, - filters=expected_filters) + filters=expected_filters, + cached=True) @ddt.data(('index', False, True), ('index', False, False), @@ -210,7 +212,8 @@ class SchedulerStatsControllerTestCase(test.TestCase): self.assertDictMatch(result, expected_result) mock_get_pools.assert_called_once_with(self.ctxt, - filters=expected_filters) + filters=expected_filters, + cached=True) @ddt.data('index', 'detail') def test_pools_with_share_type_not_found(self, action): @@ -271,7 +274,8 @@ class SchedulerStatsControllerTestCase(test.TestCase): self.assertDictMatch(result, expected) mock_get_pools.assert_called_once_with(self.ctxt, - filters=expected_filters) + filters=expected_filters, + cached=True) self.mock_policy_check.assert_called_once_with( self.ctxt, self.resource_name, 'index') @@ -323,7 +327,8 @@ class SchedulerStatsControllerTestCase(test.TestCase): } self.assertDictMatch(expected, result) - mock_get_pools.assert_called_once_with(self.ctxt, filters={}) + mock_get_pools.assert_called_once_with(self.ctxt, filters={}, + cached=True) self.mock_policy_check.assert_called_once_with( self.ctxt, self.resource_name, 'detail') diff --git a/manila/tests/scheduler/test_manager.py b/manila/tests/scheduler/test_manager.py index 5b341e2104..84399030f7 100644 --- a/manila/tests/scheduler/test_manager.py +++ b/manila/tests/scheduler/test_manager.py @@ -224,7 +224,8 @@ class SchedulerManagerTestCase(test.TestCase): result = self.manager.get_pools(self.context, filters='fake_filters') - mock_get_pools.assert_called_once_with(self.context, 'fake_filters') + mock_get_pools.assert_called_once_with(self.context, 'fake_filters', + False) self.assertEqual('fake_pools', result) @mock.patch.object(db, 'share_group_update', mock.Mock()) diff --git a/manila/tests/scheduler/test_rpcapi.py b/manila/tests/scheduler/test_rpcapi.py index 96e772efe4..effdaf310a 100644 --- a/manila/tests/scheduler/test_rpcapi.py +++ b/manila/tests/scheduler/test_rpcapi.py @@ -88,7 +88,7 @@ class SchedulerRpcAPITestCase(test.TestCase): self._test_scheduler_api('get_pools', rpc_method='call', filters=None, - version='1.1') + version='1.9') def test_create_share_group(self): self._test_scheduler_api('create_share_group', diff --git a/releasenotes/notes/bug-1804659-speed-up-pools-detail-18f539a96042099a.yaml b/releasenotes/notes/bug-1804659-speed-up-pools-detail-18f539a96042099a.yaml new file mode 100644 index 0000000000..f09893080d --- /dev/null +++ b/releasenotes/notes/bug-1804659-speed-up-pools-detail-18f539a96042099a.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Added caching of host state map to speed up calls for + scheduler-stats/pools/detail.