Add configurability to polling.yaml.

The ceilometer compute agent uses the default polling.yaml
from the installed packages without the ability to configure its contents.

This change adds two configuration options: 'polling-interval' and
'enable-all-pollsters', borrowing from the implementation in
charm-ceilometer. We start off with a limited set of meters as before
and if these are not enough, the user can set 'enable-all-pollsters' to
'true' to collect all available 'Pollster' metrics as listed in the
documentation [1].

Verification:
I tested this change on a cluster built from the OpenStack base bundle
and the ceilometer and gnocchi charms. I confirmed that extra metrics
that originate from the Compute Agent pollster (e.g. disk.device.read.latency)
are available in gnocchi after setting 'enable-all-pollsters' to true.

[1] https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html

Closes-Bug: #1914442
Change-Id: I21c9a93e7dd91bced9365e44f3e6a5315976c3bb

config.yaml

@@ -22,6 +22,19 @@ options:
       Openstack mostly defaults to using public endpoints for
       internal communication between services. If set to True this option will
       configure services to use internal endpoints where possible.
+  polling-interval:
+    type: int
+    default: 300
+    description: |
+      Number of seconds between Ceilometer compute agent pollster collections. This
+      setting only takes effect from Queens onwards.
+  enable-all-pollsters:
+    type: boolean
+    default: False
+    description: |
+      The default ceilometer pollster collection runs a limited set of pollsters.
+      Enable this to run all the available pollsters. This setting only takes effect
+      from Queens onwards.
   rabbit-user:
     type: string
     default: ceilometer

hooks/ceilometer_contexts.py

@@ -16,6 +16,7 @@ import base64
 import os
 
 from charmhelpers.core.hookenv import (
+    config,
     relation_ids,
     relation_get,
     related_units,
@@ -84,3 +85,13 @@ class CeilometerServiceContext(OSContextGenerator):
                             conf['rabbit_ssl_ca'] = ca_path
                     return conf
         return {}
+
+
+class CeilometerAgentContext(OSContextGenerator):
+    def __call__(self):
+        ctxt = {
+            'polling_interval': int(config('polling-interval')),
+            'enable_all_pollsters': config('enable-all-pollsters'),
+        }
+
+        return ctxt

hooks/ceilometer_utils.py

@@ -18,6 +18,7 @@ from charmhelpers.contrib.openstack import (
     templating,
 )
 from ceilometer_contexts import (
+    CeilometerAgentContext,
     CeilometerServiceContext,
 )
 from charmhelpers.contrib.openstack.utils import (
@@ -49,6 +50,7 @@ from charmhelpers.fetch import (
 
 CEILOMETER_CONF_DIR = "/etc/ceilometer"
 CEILOMETER_CONF = "%s/ceilometer.conf" % CEILOMETER_CONF_DIR
+POLLING_CONF = "%s/polling.yaml" % CEILOMETER_CONF_DIR
 
 CEILOMETER_AGENT_SERVICES = ['ceilometer-agent-compute']
 
@@ -95,6 +97,17 @@ CONFIG_FILES = {
         'services': CEILOMETER_AGENT_SERVICES
     },
 }
+
+QUEENS_CONFIG_FILES = deepcopy(CONFIG_FILES)
+QUEENS_CONFIG_FILES.update({
+    POLLING_CONF: {
+        'hook_contexts': [
+            CeilometerAgentContext()
+        ],
+        'services': CEILOMETER_AGENT_SERVICES
+    }
+})
+
 TEMPLATES = 'templates'
 
 REQUIRED_INTERFACES = {
@@ -117,8 +130,13 @@ def register_configs():
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
 
-    for conf in CONFIG_FILES:
+    if CompareOpenStackReleases(release) >= 'queens':
-        configs.register(conf, CONFIG_FILES[conf]['hook_contexts'])
+        _config_files = QUEENS_CONFIG_FILES
+    else:
+        _config_files = CONFIG_FILES
+
+    for conf in _config_files:
+        configs.register(conf, _config_files[conf]['hook_contexts'])
 
     if enable_memcache(release=release):
         configs.register(
@@ -192,8 +210,14 @@ def restart_map():
     '''
     release = (get_os_codename_package('ceilometer-common', fatal=False) or
                'icehouse')
+
+    if CompareOpenStackReleases(release) >= 'queens':
+        _config_files = QUEENS_CONFIG_FILES
+    else:
+        _config_files = CONFIG_FILES
+
     _map = {}
-    for f, ctxt in CONFIG_FILES.items():
+    for f, ctxt in _config_files.items():
         svcs = []
         for svc in ctxt['services']:
             svcs.append(svc)

templates/polling.yaml

@@ -0,0 +1,31 @@
+---
+sources:
+  - name: juju_pollsters
+    interval: {{ polling_interval }}
+    meters:
+{%- if enable_all_pollsters %}
+      - "*"
+{%- else %}
+      - cpu
+      - cpu_l3_cache
+      - memory.usage
+      - network.incoming.bytes
+      - network.incoming.packets
+      - network.outgoing.bytes
+      - network.outgoing.packets
+      - disk.device.read.bytes
+      - disk.device.read.requests
+      - disk.device.write.bytes
+      - disk.device.write.requests
+      - hardware.cpu.util
+      - hardware.memory.used
+      - hardware.memory.total
+      - hardware.memory.buffer
+      - hardware.memory.cached
+      - hardware.memory.swap.avail
+      - hardware.memory.swap.total
+      - hardware.system_stats.io.outgoing.blocks
+      - hardware.system_stats.io.incoming.blocks
+      - hardware.network.ip.incoming.datagrams
+      - hardware.network.ip.outgoing.datagrams
+{%- endif %}

unit_tests/test_ceilometer_contexts.py

@@ -16,6 +16,7 @@ import ceilometer_contexts as contexts
 from test_utils import CharmTestCase
 
 TO_PATCH = [
+    'config',
     'relation_get',
     'relation_ids',
     'related_units',
@@ -26,6 +27,7 @@ class CeilometerContextsTest(CharmTestCase):
 
     def setUp(self):
         super(CeilometerContextsTest, self).setUp(contexts, TO_PATCH)
+        self.config.side_effect = self.test_config.get
         self.relation_get.side_effect = self.test_relation.get
 
     def tearDown(self):
@@ -65,3 +67,23 @@ class CeilometerContextsTest(CharmTestCase):
     def test_ceilometer_service_context_not_related(self):
         self.relation_ids.return_value = []
         self.assertEqual(contexts.CeilometerServiceContext()(), {})
+
+    def test_ceilometer_context(self):
+        self.assertEqual(contexts.CeilometerAgentContext()(), {
+            'polling_interval': 300,
+            'enable_all_pollsters': False,
+        })
+
+    def test_ceilometer_context_enable_all_pollsters(self):
+        self.test_config.set('enable-all-pollsters', True)
+        self.assertEqual(contexts.CeilometerAgentContext()(), {
+            'polling_interval': 300,
+            'enable_all_pollsters': True,
+        })
+
+    def test_ceilometer_context_polling_interval(self):
+        self.test_config.set('polling-interval', 600)
+        self.assertEqual(contexts.CeilometerAgentContext()(), {
+            'polling_interval': 600,
+            'enable_all_pollsters': False,
+        })

unit_tests/test_ceilometer_utils.py

@@ -53,6 +53,7 @@ class CeilometerUtilsTest(CharmTestCase):
         super(CeilometerUtilsTest, self).tearDown()
 
     def test_register_configs(self):
+        self.get_os_codename_package.return_value = 'icehouse'
         self.enable_memcache.return_value = False
         configs = utils.register_configs()
         registered_configs = [c[0][0] for c in configs.register.call_args_list]
@@ -60,6 +61,7 @@ class CeilometerUtilsTest(CharmTestCase):
         self.assertFalse(utils.MEMCACHED_CONF in registered_configs)
 
     def test_register_configs_newton(self):
+        self.get_os_codename_package.return_value = 'newton'
         self.enable_memcache.return_value = True
         configs = utils.register_configs()
         registered_configs = [c[0][0] for c in configs.register.call_args_list]
@@ -67,18 +69,31 @@ class CeilometerUtilsTest(CharmTestCase):
             self.assertTrue(config in registered_configs)
 
     def test_restart_map(self):
+        self.get_os_codename_package.return_value = 'icehouse'
         self.enable_memcache.return_value = False
         restart_map = utils.restart_map()
-        self.assertEqual(restart_map,
+        self.assertEqual(restart_map, {
-                         {'/etc/ceilometer/ceilometer.conf': [
+            '/etc/ceilometer/ceilometer.conf': ['ceilometer-agent-compute'],
-                             'ceilometer-agent-compute']})
+        })
 
     def test_restart_map_newton(self):
+        self.get_os_codename_package.return_value = 'newton'
         self.enable_memcache.return_value = True
         restart_map = utils.restart_map()
         expect = {
             '/etc/ceilometer/ceilometer.conf': ['ceilometer-agent-compute'],
-            '/etc/memcached.conf': ['memcached']}
+            '/etc/memcached.conf': ['memcached'],
+        }
+        self.assertEqual(restart_map, expect)
+
+    def test_restart_map_queens(self):
+        self.get_os_codename_package.return_value = 'queens'
+        restart_map = utils.restart_map()
+        expect = {
+            '/etc/ceilometer/ceilometer.conf': ['ceilometer-agent-compute'],
+            '/etc/memcached.conf': ['memcached'],
+            '/etc/ceilometer/polling.yaml': ['ceilometer-agent-compute'],
+        }
         self.assertEqual(restart_map, expect)
 
     def test_assess_status(self):