Allow using file to override metric map

Override the metric map of each datasource as soon as it is created by
the manager. This override comes from a file whose path is provided by
a setting in config file.

Loading at creation time allows the correct datasource be used when
get_backend is called, this allows loading a datasource whose metric
names get updated outside the watcher's codebase.

The function 'load_metric_map' returns empty-dict in any error case.
Also in case the file is empty where safe_load is unable finds any
yaml documents, it will return None. [1]

Some minor refactoring in the test_manager file for readability and
added tests for file load and metric override.

1 - https://pyyaml.org/wiki/PyYAMLDocumentation

Change-Id: I1df16245f4c7dfd34066f3ab0553cd67154faa58
Implements: blueprint file-based-metric-map

releasenotes/notes/file-based-metric-map-c2af62b5067895df.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Allow using file to override metric map. Override the metric map of
+    each datasource as soon as it is created by the manager. This override
+    comes from a file whose path is provided by a setting in config file.
+    The setting is `watcher_decision_engine/metric_map_path`.  The file
+    contains a map per datasource whose keys are the metric names as
+    recognized by watcher and the value is the real name of the metric
+    in the datasource. This setting defaults to `/etc/watcher/metric_map.yaml`,
+    and presence of this file is optional.

watcher/conf/decision_engine.py

@@ -55,8 +55,20 @@ WATCHER_DECISION_ENGINE_OPTS = [
     cfg.IntOpt('check_periodic_interval',
                default=30 * 60,
                mutable=True,
-               help='Interval (in seconds) for checking action plan expiry.')
-]
+               help='Interval (in seconds) for checking action plan expiry.'),
+    cfg.StrOpt('metric_map_path',
+               default='/etc/watcher/metric_map.yaml',
+               help='Path to metric map yaml formatted file. '
+                    ' '
+                    'The file contains a map per datasource whose keys '
+                    'are the metric names as recognized by watcher and the '
+                    'value is the real name of the metric in the datasource. '
+                    'For example:: \n\n'
+                    '  monasca:\n'
+                    '    instance_cpu_usage: VM_CPU\n'
+                    '  gnocchi:\n'
+                    '    instance_cpu_usage: cpu_vm_util\n\n'
+                    'This file is optional.')]
 
 WATCHER_CONTINUOUS_OPTS = [
     cfg.IntOpt('continuous_audit_interval',

watcher/datasources/manager.py

@@ -13,13 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import yaml
+
 from collections import OrderedDict
+from oslo_config import cfg
+from oslo_log import log
 
 from watcher.common import exception
 from watcher.datasources import ceilometer as ceil
 from watcher.datasources import gnocchi as gnoc
 from watcher.datasources import monasca as mon
 
+LOG = log.getLogger(__name__)
+
 
 class DataSourceManager(object):
 
@@ -38,6 +45,15 @@ class DataSourceManager(object):
         self._ceilometer = None
         self._monasca = None
         self._gnocchi = None
+
+        metric_map_path = cfg.CONF.watcher_decision_engine.metric_map_path
+        metrics_from_file = self.load_metric_map(metric_map_path)
+        for ds, mp in self.metric_map.items():
+            try:
+                self.metric_map[ds].update(metrics_from_file.get(ds, {}))
+            except KeyError:
+                msgargs = (ds, self.metric_map.keys())
+                LOG.warning('Invalid Datasource: %s. Allowed: %s ', *msgargs)
         self.datasources = self.config.datasources
 
     @property
@@ -82,7 +98,23 @@ class DataSourceManager(object):
                 # Try to use a specific datasource but attempt additional
                 # datasources upon exceptions (if config has more datasources)
                 try:
-                    return getattr(self, datasource)
+                    ds = getattr(self, datasource)
+                    ds.METRIC_MAP.update(self.metric_map[ds.NAME])
+                    return ds
                 except Exception:
                     pass
         raise exception.NoSuchMetric()
+
+    def load_metric_map(self, file_path):
+        """Load metrics from the metric_map_path"""
+        if file_path and os.path.exists(file_path):
+            with open(file_path, 'r') as f:
+                try:
+                    ret = yaml.safe_load(f.read())
+                    # return {} if the file is empty
+                    return ret if ret else {}
+                except yaml.YAMLError as e:
+                    LOG.warning('Could not load %s: %s', file_path, e)
+                return {}
+        else:
+            return {}

watcher/tests/datasources/test_manager.py

@@ -16,44 +16,72 @@
 
 import mock
 
+from mock import MagicMock
+
 from watcher.common import exception
 from watcher.datasources import gnocchi
 from watcher.datasources import manager as ds_manager
+from watcher.datasources import monasca
 from watcher.tests import base
 
 
 class TestDataSourceManager(base.BaseTestCase):
 
+    def _dsm_config(self, **kwargs):
+        dss = ['gnocchi', 'ceilometer', 'monasca']
+        opts = dict(datasources=dss, metric_map_path=None)
+        opts.update(kwargs)
+        return MagicMock(**opts)
+
+    def _dsm(self, **kwargs):
+        opts = dict(config=self._dsm_config(), osc=mock.MagicMock())
+        opts.update(kwargs)
+        return ds_manager.DataSourceManager(**opts)
+
     def test_get_backend(self):
-        manager = ds_manager.DataSourceManager(
-            config=mock.MagicMock(
-                datasources=['gnocchi', 'ceilometer', 'monasca']),
-            osc=mock.MagicMock())
+        manager = self._dsm()
         backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
         self.assertEqual(backend, manager.gnocchi)
 
     def test_get_backend_order(self):
-        manager = ds_manager.DataSourceManager(
-            config=mock.MagicMock(
-                datasources=['monasca', 'ceilometer', 'gnocchi']),
-            osc=mock.MagicMock())
+        dss = ['monasca', 'ceilometer', 'gnocchi']
+        dsmcfg = self._dsm_config(datasources=dss)
+        manager = self._dsm(config=dsmcfg)
         backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
         self.assertEqual(backend, manager.monasca)
 
     def test_get_backend_wrong_metric(self):
-        manager = ds_manager.DataSourceManager(
-            config=mock.MagicMock(
-                datasources=['gnocchi', 'ceilometer', 'monasca']),
-            osc=mock.MagicMock())
+        manager = self._dsm()
         self.assertRaises(exception.NoSuchMetric, manager.get_backend,
                           ['host_cpu', 'instance_cpu_usage'])
 
     @mock.patch.object(gnocchi, 'GnocchiHelper')
     def test_get_backend_error_datasource(self, m_gnocchi):
         m_gnocchi.side_effect = exception.DataSourceNotAvailable
-        manager = ds_manager.DataSourceManager(
-            config=mock.MagicMock(
-                datasources=['gnocchi', 'ceilometer', 'monasca']),
-            osc=mock.MagicMock())
+        manager = self._dsm()
         backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
         self.assertEqual(backend, manager.ceilometer)
+
+    def test_metric_file_path_not_exists(self):
+        manager = self._dsm()
+        expected = ds_manager.DataSourceManager.metric_map
+        actual = manager.metric_map
+        self.assertEqual(expected, actual)
+        self.assertEqual({}, manager.load_metric_map('/nope/nope/nope.yaml'))
+
+    def test_metric_file_metric_override(self):
+        path = 'watcher.datasources.manager.DataSourceManager.load_metric_map'
+        retval = {
+            monasca.MonascaHelper.NAME: {"host_airflow": "host_fnspid"}
+        }
+        with mock.patch(path, return_value=retval):
+            dsmcfg = self._dsm_config(datasources=['monasca'])
+            manager = self._dsm(config=dsmcfg)
+            backend = manager.get_backend(['host_cpu_usage'])
+            self.assertEqual("host_fnspid", backend.METRIC_MAP['host_airflow'])
+
+    def test_metric_file_invalid_ds(self):
+        with mock.patch('yaml.safe_load') as mo:
+            mo.return_value = {"newds": {"metric_one": "i_am_metric_one"}}
+            mgr = self._dsm()
+            self.assertNotIn('newds', mgr.metric_map.keys())