Merge "Allow for global datasources preference from config"

doc/source/contributor/plugin/strategy-plugin.rst

@@ -285,8 +285,15 @@ The following code snippet shows how datasource_backend is defined:
     @property
     def datasource_backend(self):
         if not self._datasource_backend:
+
+            # Load the global preferred datasources order but override it
+            # if the strategy has a specific datasources config
+            datasources = CONF.watcher_datasources
+            if self.config.datasources:
+                datasources = self.config
+
             self._datasource_backend = ds_manager.DataSourceManager(
-                config=self.config,
+                config=datasources,
                 osc=self.osc
             ).get_backend(self.DATASOURCE_METRICS)
         return self._datasource_backend

releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Watcher now supports configuring which datasource to use and in which
+    order. This configuration is done by specifying datasources in the
+    watcher_datasources section:
+
+    - ``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``
+
+    Specific strategies can override this order and use datasources which
+    are not listed in the global preference.

watcher/common/clients.py

@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from oslo_config import cfg
 
 from cinderclient import client as ciclient
 from glanceclient import client as glclient
@@ -23,15 +24,13 @@ from novaclient import client as nvclient
 
 from watcher.common import exception
 
-from watcher import conf
-
 try:
     from ceilometerclient import client as ceclient
     HAS_CEILCLIENT = True
 except ImportError:
     HAS_CEILCLIENT = False
 
-CONF = conf.CONF
+CONF = cfg.CONF
 
 _CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
 

watcher/common/exception.py

@@ -26,16 +26,15 @@ import functools
 import sys
 
 from keystoneclient import exceptions as keystone_exceptions
+from oslo_config import cfg
 from oslo_log import log
 import six
 
 from watcher._i18n import _
 
-from watcher import conf
-
 LOG = log.getLogger(__name__)
 
-CONF = conf.CONF
+CONF = cfg.CONF
 
 
 def wrap_keystone_exception(func):

watcher/common/utils.py

@@ -24,6 +24,7 @@ import string
 from croniter import croniter
 
 from jsonschema import validators
+from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import strutils
 from oslo_utils import uuidutils
@@ -31,9 +32,7 @@ import six
 
 from watcher.common import exception
 
-from watcher import conf
+CONF = cfg.CONF
-
-CONF = conf.CONF
 
 LOG = log.getLogger(__name__)
 

watcher/conf/__init__.py

@@ -25,6 +25,7 @@ from watcher.conf import ceilometer_client
 from watcher.conf import cinder_client
 from watcher.conf import clients_auth
 from watcher.conf import collector
+from watcher.conf import datasources
 from watcher.conf import db
 from watcher.conf import decision_engine
 from watcher.conf import exception
@@ -44,6 +45,7 @@ service.register_opts(CONF)
 api.register_opts(CONF)
 paths.register_opts(CONF)
 exception.register_opts(CONF)
+datasources.register_opts(CONF)
 db.register_opts(CONF)
 planner.register_opts(CONF)
 applier.register_opts(CONF)

watcher/conf/datasources.py

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+from watcher.datasources import manager
+
+datasources = cfg.OptGroup(name='watcher_datasources',
+                           title='Configuration Options for watcher'
+                                 ' datasources')
+
+possible_datasources = list(manager.DataSourceManager.metric_map.keys())
+
+DATASOURCES_OPTS = [
+    cfg.ListOpt("datasources",
+                help="Datasources to use in order to query the needed metrics."
+                     " If one of strategy metric is not available in the first"
+                     " datasource, the next datasource will be chosen. This is"
+                     " the default for all strategies unless a strategy has a"
+                     " specific override.",
+                item_type=cfg.types.String(choices=possible_datasources),
+                default=possible_datasources)
+    ]
+
+
+def register_opts(conf):
+    conf.register_group(datasources)
+    conf.register_opts(DATASOURCES_OPTS, group=datasources)
+
+
+def list_opts():
+    return [('watcher_datasources', DATASOURCES_OPTS)]

watcher/datasources/ceilometer.py

@@ -25,7 +25,7 @@ from oslo_utils import timeutils
 from watcher._i18n import _
 from watcher.common import clients
 from watcher.common import exception
-from watcher.datasource import base
+from watcher.datasources import base
 
 
 LOG = log.getLogger(__name__)

watcher/datasources/gnocchi.py

@@ -26,7 +26,7 @@ from oslo_log import log
 from watcher.common import clients
 from watcher.common import exception
 from watcher.common import utils as common_utils
-from watcher.datasource import base
+from watcher.datasources import base
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)

watcher/datasources/manager.py

@@ -13,25 +13,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import OrderedDict
+
 from watcher.common import exception
-from watcher.datasource import ceilometer as ceil
+from watcher.datasources import ceilometer as ceil
-from watcher.datasource import gnocchi as gnoc
+from watcher.datasources import gnocchi as gnoc
-from watcher.datasource import monasca as mon
+from watcher.datasources import monasca as mon
 
 
 class DataSourceManager(object):
 
+    metric_map = OrderedDict([
+        (gnoc.GnocchiHelper.NAME, gnoc.GnocchiHelper.METRIC_MAP),
+        (ceil.CeilometerHelper.NAME, ceil.CeilometerHelper.METRIC_MAP),
+        (mon.MonascaHelper.NAME, mon.MonascaHelper.METRIC_MAP),
+    ])
+    """Dictionary with all possible datasources, dictionary order is the default
+    order for attempting to use datasources
+    """
+
     def __init__(self, config=None, osc=None):
         self.osc = osc
         self.config = config
         self._ceilometer = None
         self._monasca = None
         self._gnocchi = None
-        self.metric_map = {
-            mon.MonascaHelper.NAME: mon.MonascaHelper.METRIC_MAP,
-            gnoc.GnocchiHelper.NAME: gnoc.GnocchiHelper.METRIC_MAP,
-            ceil.CeilometerHelper.NAME: ceil.CeilometerHelper.METRIC_MAP
-        }
         self.datasources = self.config.datasources
 
     @property
@@ -73,5 +79,10 @@ class DataSourceManager(object):
                         no_metric = True
                         break
             if not no_metric:
-                return getattr(self, datasource)
+                # Try to use a specific datasource but attempt additional
+                # datasources upon exceptions (if config has more datasources)
+                try:
+                    return getattr(self, datasource)
+                except Exception:
+                    pass
         raise exception.NoSuchMetric()

watcher/datasources/monasca.py

@@ -22,7 +22,7 @@ from monascaclient import exc
 
 from watcher.common import clients
 from watcher.common import exception
-from watcher.datasource import base
+from watcher.datasources import base
 
 
 class MonascaHelper(base.DataSourceBase):

watcher/decision_engine/strategy/strategies/actuation.py

@@ -80,6 +80,12 @@ class Actuator(base.UnclassifiedStrategy):
             ]
         }
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
     @property
     def actions(self):
         return self.input_parameters.get('actions', [])

watcher/decision_engine/strategy/strategies/base.py

@@ -48,7 +48,7 @@ from watcher.common import context
 from watcher.common import exception
 from watcher.common.loader import loadable
 from watcher.common import utils
-from watcher.datasource import manager as ds_manager
+from watcher.datasources import manager as ds_manager
 from watcher.decision_engine.loading import default as loading
 from watcher.decision_engine.model.collector import manager
 from watcher.decision_engine.solution import default
@@ -130,6 +130,8 @@ class BaseStrategy(loadable.Loadable):
     """
 
     DATASOURCE_METRICS = []
+    """Contains all metrics the strategy requires from a datasource to properly
+    execute"""
 
     MIGRATION = "migrate"
 
@@ -199,7 +201,18 @@ class BaseStrategy(loadable.Loadable):
         :return: A list of configuration options relative to this Loadable
         :rtype: list of :class:`oslo_config.cfg.Opt` instances
         """
-        return []
+
+        datasources_ops = list(ds_manager.DataSourceManager.metric_map.keys())
+
+        return [
+            cfg.ListOpt(
+                "datasources",
+                help="Datasources to use in order to query the needed metrics."
+                     " This option overrides the global preference."
+                     " options: {0}".format(datasources_ops),
+                item_type=cfg.types.String(choices=datasources_ops),
+                default=None)
+        ]
 
     @abc.abstractmethod
     def pre_execute(self):
@@ -343,8 +356,15 @@ class BaseStrategy(loadable.Loadable):
     @property
     def datasource_backend(self):
         if not self._datasource_backend:
+
+            # Load the global preferred datasources order but override it
+            # if the strategy has a specific datasources config
+            datasources = CONF.watcher_datasources
+            if self.config.datasources:
+                datasources = self.config
+
             self._datasource_backend = ds_manager.DataSourceManager(
-                config=self.config,
+                config=datasources,
                 osc=self.osc
             ).get_backend(self.DATASOURCE_METRICS)
         return self._datasource_backend
@@ -443,6 +463,12 @@ class DummyBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "dummy"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class UnclassifiedStrategy(BaseStrategy):
@@ -500,6 +526,12 @@ class SavingEnergyBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "saving_energy"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class ZoneMigrationBaseStrategy(BaseStrategy):
@@ -508,6 +540,12 @@ class ZoneMigrationBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "hardware_maintenance"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class HostMaintenanceBaseStrategy(BaseStrategy):
@@ -517,3 +555,9 @@ class HostMaintenanceBaseStrategy(BaseStrategy):
     @classmethod
     def get_goal_name(cls):
         return "cluster_maintaining"
+
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -171,19 +171,11 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
 
     @classmethod
     def get_config_opts(cls):
-        return [
+        return super(BasicConsolidation, cls).get_config_opts() + [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
             cfg.BoolOpt(
-                "check_optimize_metadata",
+                'check_optimize_metadata',
-                help="Check optimize metadata field in instance before "
+                help='Check optimize metadata field in instance before'
-                     "migration",
+                     ' migration',
                 default=False),
         ]
 

watcher/decision_engine/strategy/strategies/noisy_neighbor.py

@@ -94,19 +94,6 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_current_and_previous_cache(self, instance):
         try:
             curr_cache = self.datasource_backend.get_instance_l3_cache_usage(

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -32,7 +32,6 @@ thermal condition (lowest outlet temperature) when the outlet temperature
 of source hosts reach a configurable threshold.
 """
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -141,19 +140,6 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy):
     def granularity(self):
         return self.input_parameters.get('granularity', 300)
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/storage_capacity_balance.py

@@ -99,7 +99,7 @@ class StorageCapacityBalance(base.WorkloadStabilizationBaseStrategy):
 
     @classmethod
     def get_config_opts(cls):
-        return [
+        return super(StorageCapacityBalance, cls).get_config_opts() + [
             cfg.ListOpt(
                 "ex_pools",
                 help="exclude pools",

watcher/decision_engine/strategy/strategies/uniform_airflow.py

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -148,19 +147,6 @@ class UniformAirflow(base.BaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -18,7 +18,6 @@
 # limitations under the License.
 #
 
-from oslo_config import cfg
 from oslo_log import log
 import six
 
@@ -138,19 +137,6 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
             }
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value,
                               element.ServiceState.DISABLED.value]

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -19,7 +19,6 @@
 
 from __future__ import division
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -130,19 +129,6 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -226,19 +226,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
             }
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def transform_instance_cpu(self, instance_load, host_vcpus):
         """Transform instance cpu utilization to overall host cpu utilization.
 

watcher/tests/conf/test_list_opts.py

@@ -29,8 +29,8 @@ class TestListOpts(base.TestCase):
         super(TestListOpts, self).setUp()
         self.base_sections = [
             'DEFAULT', 'api', 'database', 'watcher_decision_engine',
-            'watcher_applier', 'watcher_planner', 'nova_client',
+            'watcher_applier', 'watcher_datasources', 'watcher_planner',
-            'glance_client', 'gnocchi_client', 'cinder_client',
+            'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
             'ceilometer_client', 'monasca_client', 'ironic_client',
             'neutron_client', 'watcher_clients_auth', 'collector']
         self.opt_sections = list(dict(opts.list_opts()).keys())

watcher/tests/datasources/test_ceilometer_helper.py

@@ -20,7 +20,7 @@ from __future__ import unicode_literals
 import mock
 
 from watcher.common import clients
-from watcher.datasource import ceilometer as ceilometer_helper
+from watcher.datasources import ceilometer as ceilometer_helper
 from watcher.tests import base
 
 

watcher/tests/datasources/test_gnocchi_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import gnocchi as gnocchi_helper
+from watcher.datasources import gnocchi as gnocchi_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/datasources/test_manager.py

@@ -17,7 +17,8 @@
 import mock
 
 from watcher.common import exception
-from watcher.datasource import manager as ds_manager
+from watcher.datasources import gnocchi
+from watcher.datasources import manager as ds_manager
 from watcher.tests import base
 
 
@@ -46,3 +47,13 @@ class TestDataSourceManager(base.BaseTestCase):
             osc=mock.MagicMock())
         self.assertRaises(exception.NoSuchMetric, manager.get_backend,
                           ['host_cpu', 'instance_cpu_usage'])
+
+    @mock.patch.object(gnocchi, 'GnocchiHelper')
+    def test_get_backend_error_datasource(self, m_gnocchi):
+        m_gnocchi.side_effect = exception.DataSourceNotAvailable
+        manager = ds_manager.DataSourceManager(
+            config=mock.MagicMock(
+                datasources=['gnocchi', 'ceilometer', 'monasca']),
+            osc=mock.MagicMock())
+        backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
+        self.assertEqual(backend, manager.ceilometer)

watcher/tests/datasources/test_monasca_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import monasca as monasca_helper
+from watcher.datasources import monasca as monasca_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/decision_engine/strategy/strategies/test_base.py

@@ -17,6 +17,7 @@
 import mock
 
 from watcher.common import exception
+from watcher.datasources import manager
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
@@ -49,6 +50,67 @@ class TestBaseStrategy(base.TestCase):
         self.strategy = strategies.DummyStrategy(config=mock.Mock())
 
 
+class TestBaseStrategyDatasource(TestBaseStrategy):
+
+    def setUp(self):
+        super(TestBaseStrategyDatasource, self).setUp()
+        self.strategy = strategies.DummyStrategy(
+            config=mock.Mock(datasources=None))
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference(self, m_conf, m_manager):
+        """Test if the global preference is used"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['gnocchi', 'monasca', 'ceilometer']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference_reverse(self, m_conf, m_manager):
+        """Test if the global preference is used with another order"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_strategy_preference_override(self, m_conf, m_manager):
+        """Test if the global preference can be overridden"""
+
+        datasources = mock.Mock(datasources=['ceilometer'])
+
+        self.strategy = strategies.DummyStrategy(
+            config=datasources)
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=datasources, osc=None)
+
+
 class TestBaseStrategyException(TestBaseStrategy):
 
     def setUp(self):