diff --git a/releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml b/releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml new file mode 100644 index 000000000..0fef96202 --- /dev/null +++ b/releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Allow using file to override metric map. Override the metric map of + each datasource as soon as it is created by the manager. This override + comes from a file whose path is provided by a setting in config file. + The setting is `watcher_decision_engine/metric_map_path`. The file + contains a map per datasource whose keys are the metric names as + recognized by watcher and the value is the real name of the metric + in the datasource. This setting defaults to `/etc/watcher/metric_map.yaml`, + and presence of this file is optional. diff --git a/watcher/conf/decision_engine.py b/watcher/conf/decision_engine.py index eb0512883..962fd8cde 100644 --- a/watcher/conf/decision_engine.py +++ b/watcher/conf/decision_engine.py @@ -55,8 +55,20 @@ WATCHER_DECISION_ENGINE_OPTS = [ cfg.IntOpt('check_periodic_interval', default=30 * 60, mutable=True, - help='Interval (in seconds) for checking action plan expiry.') -] + help='Interval (in seconds) for checking action plan expiry.'), + cfg.StrOpt('metric_map_path', + default='/etc/watcher/metric_map.yaml', + help='Path to metric map yaml formatted file. ' + ' ' + 'The file contains a map per datasource whose keys ' + 'are the metric names as recognized by watcher and the ' + 'value is the real name of the metric in the datasource. ' + 'For example:: \n\n' + ' monasca:\n' + ' instance_cpu_usage: VM_CPU\n' + ' gnocchi:\n' + ' instance_cpu_usage: cpu_vm_util\n\n' + 'This file is optional.')] WATCHER_CONTINUOUS_OPTS = [ cfg.IntOpt('continuous_audit_interval', diff --git a/watcher/datasources/manager.py b/watcher/datasources/manager.py index 1162e7bed..77a1feb81 100644 --- a/watcher/datasources/manager.py +++ b/watcher/datasources/manager.py @@ -13,13 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import yaml + from collections import OrderedDict +from oslo_config import cfg +from oslo_log import log from watcher.common import exception from watcher.datasources import ceilometer as ceil from watcher.datasources import gnocchi as gnoc from watcher.datasources import monasca as mon +LOG = log.getLogger(__name__) + class DataSourceManager(object): @@ -38,6 +45,15 @@ class DataSourceManager(object): self._ceilometer = None self._monasca = None self._gnocchi = None + + metric_map_path = cfg.CONF.watcher_decision_engine.metric_map_path + metrics_from_file = self.load_metric_map(metric_map_path) + for ds, mp in self.metric_map.items(): + try: + self.metric_map[ds].update(metrics_from_file.get(ds, {})) + except KeyError: + msgargs = (ds, self.metric_map.keys()) + LOG.warning('Invalid Datasource: %s. Allowed: %s ', *msgargs) self.datasources = self.config.datasources @property @@ -82,7 +98,23 @@ class DataSourceManager(object): # Try to use a specific datasource but attempt additional # datasources upon exceptions (if config has more datasources) try: - return getattr(self, datasource) + ds = getattr(self, datasource) + ds.METRIC_MAP.update(self.metric_map[ds.NAME]) + return ds except Exception: pass raise exception.NoSuchMetric() + + def load_metric_map(self, file_path): + """Load metrics from the metric_map_path""" + if file_path and os.path.exists(file_path): + with open(file_path, 'r') as f: + try: + ret = yaml.safe_load(f.read()) + # return {} if the file is empty + return ret if ret else {} + except yaml.YAMLError as e: + LOG.warning('Could not load %s: %s', file_path, e) + return {} + else: + return {} diff --git a/watcher/tests/datasources/test_manager.py b/watcher/tests/datasources/test_manager.py index 19bb40088..7494bd3d5 100644 --- a/watcher/tests/datasources/test_manager.py +++ b/watcher/tests/datasources/test_manager.py @@ -16,44 +16,72 @@ import mock +from mock import MagicMock + from watcher.common import exception from watcher.datasources import gnocchi from watcher.datasources import manager as ds_manager +from watcher.datasources import monasca from watcher.tests import base class TestDataSourceManager(base.BaseTestCase): + def _dsm_config(self, **kwargs): + dss = ['gnocchi', 'ceilometer', 'monasca'] + opts = dict(datasources=dss, metric_map_path=None) + opts.update(kwargs) + return MagicMock(**opts) + + def _dsm(self, **kwargs): + opts = dict(config=self._dsm_config(), osc=mock.MagicMock()) + opts.update(kwargs) + return ds_manager.DataSourceManager(**opts) + def test_get_backend(self): - manager = ds_manager.DataSourceManager( - config=mock.MagicMock( - datasources=['gnocchi', 'ceilometer', 'monasca']), - osc=mock.MagicMock()) + manager = self._dsm() backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage']) self.assertEqual(backend, manager.gnocchi) def test_get_backend_order(self): - manager = ds_manager.DataSourceManager( - config=mock.MagicMock( - datasources=['monasca', 'ceilometer', 'gnocchi']), - osc=mock.MagicMock()) + dss = ['monasca', 'ceilometer', 'gnocchi'] + dsmcfg = self._dsm_config(datasources=dss) + manager = self._dsm(config=dsmcfg) backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage']) self.assertEqual(backend, manager.monasca) def test_get_backend_wrong_metric(self): - manager = ds_manager.DataSourceManager( - config=mock.MagicMock( - datasources=['gnocchi', 'ceilometer', 'monasca']), - osc=mock.MagicMock()) + manager = self._dsm() self.assertRaises(exception.NoSuchMetric, manager.get_backend, ['host_cpu', 'instance_cpu_usage']) @mock.patch.object(gnocchi, 'GnocchiHelper') def test_get_backend_error_datasource(self, m_gnocchi): m_gnocchi.side_effect = exception.DataSourceNotAvailable - manager = ds_manager.DataSourceManager( - config=mock.MagicMock( - datasources=['gnocchi', 'ceilometer', 'monasca']), - osc=mock.MagicMock()) + manager = self._dsm() backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage']) self.assertEqual(backend, manager.ceilometer) + + def test_metric_file_path_not_exists(self): + manager = self._dsm() + expected = ds_manager.DataSourceManager.metric_map + actual = manager.metric_map + self.assertEqual(expected, actual) + self.assertEqual({}, manager.load_metric_map('/nope/nope/nope.yaml')) + + def test_metric_file_metric_override(self): + path = 'watcher.datasources.manager.DataSourceManager.load_metric_map' + retval = { + monasca.MonascaHelper.NAME: {"host_airflow": "host_fnspid"} + } + with mock.patch(path, return_value=retval): + dsmcfg = self._dsm_config(datasources=['monasca']) + manager = self._dsm(config=dsmcfg) + backend = manager.get_backend(['host_cpu_usage']) + self.assertEqual("host_fnspid", backend.METRIC_MAP['host_airflow']) + + def test_metric_file_invalid_ds(self): + with mock.patch('yaml.safe_load') as mo: + mo.return_value = {"newds": {"metric_one": "i_am_metric_one"}} + mgr = self._dsm() + self.assertNotIn('newds', mgr.metric_map.keys())