Merge "prometheus: Support overriding address of scrape targets"

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -24,7 +24,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ prometheus_port }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -36,7 +36,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-node-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -56,7 +56,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-mysqld-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -69,7 +69,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['loadbalancer'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -82,7 +82,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['rabbitmq'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -95,7 +95,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-memcached-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -108,7 +108,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-cadvisor"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -121,7 +121,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['fluentd'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -159,7 +159,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-elasticsearch-exporter"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -204,7 +204,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-libvirt-exporter"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -220,7 +220,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["etcd"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -232,7 +232,8 @@ scrape_configs:
   - job_name: ironic_prometheus_exporter
     static_configs:
 {% for host in groups['ironic-conductor'] %}
-      - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"]
+      - targets:
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -245,7 +246,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-alertmanager'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -257,7 +258,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["loadbalancer"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['proxysql_prometheus_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['proxysql_prometheus_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -270,7 +271,7 @@ alerting:
     - static_configs:
 {% for host in groups["prometheus-alertmanager"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -243,3 +243,16 @@ For example:
       labels:
         instance: host1
 
+Target address
+~~~~~~~~~~~~~~
+
+By default, Prometheus server uses the IP of the API interface of scrape
+targets when collecting metrics. This may be overridden by setting
+``prometheus_target_address`` as a host variable. The value of this host
+variable must be a valid IPv4 or IPv6 address.
+
+Prometheus server is one of the few instances where we need to know IP
+addresses of all other hosts in the cloud. Being able to specify these via
+``prometheus_target_address`` allows us to operate when facts are not available
+for all hosts. This could be due to some hosts being unreachable or having
+previously failed.

kolla_ansible/kolla_address.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import ipaddress
+
 from jinja2.filters import pass_context
 from jinja2.runtime import Undefined
 
@@ -22,7 +24,7 @@ from kolla_ansible.helpers import _call_bool_filter
 
 
 @pass_context
-def kolla_address(context, network_name, hostname=None):
+def kolla_address(context, network_name, hostname=None, override_var=None):
     """returns IP address on the requested network
 
     The output is affected by '<network_name>_*' variables:
@@ -33,6 +35,8 @@ def kolla_address(context, network_name, hostname=None):
     :param network_name: string denoting the name of the network to get IP
                          address for, e.g. 'api'
     :param hostname: to override host which address is retrieved for
+    :param override_var: optional name of a host variable that can be used
+                         to override the IP address
     :returns: string with IP address
     """
 
@@ -58,6 +62,25 @@ def kolla_address(context, network_name, hostname=None):
 
     del hostvars  # remove for clarity (no need for other hosts)
 
+    if override_var is not None:
+        if override_var in host:
+            try:
+                # Use ipaddress to test IPv4/6 validity of
+                # the string returned from override_var and
+                # return the string-formatted, valid IP address
+                ip = ipaddress.ip_address(host[override_var])
+                return format(ip)
+            except ValueError:
+                # Catch ValueError from ipaddress and make the
+                # output more useful for operators
+                raise FilterError("variable '{override_var}' for "
+                                  "host '{hostname}' is set to "
+                                  "'{value}', which is not a valid "
+                                  "IPv4 or IPv6 address"
+                                  .format(override_var=override_var,
+                                          hostname=hostname,
+                                          value=host[override_var]))
+
     # NOTE(yoctozepto): variable "host" will *not* return Undefined
     # same applies to all its children (act like plain dictionary)
 

kolla_ansible/tests/unit/test_address_filters.py

@@ -325,6 +325,71 @@ class TestKollaAddressFilter(unittest.TestCase):
         })
         self.assertEqual(addr, kolla_address(context, 'api'))
 
+    def test_override_var_valid_ipv4(self):
+        addr = '192.0.2.1'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', override_var=override_var))
+
+    def test_override_var_valid_ipv6(self):
+        addr = 'fd::'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', override_var=override_var))
+
+    def test_override_var_invalid(self):
+        addr = 'this-is-an-fqdn.example.com'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertRaises(
+            FilterError, kolla_address, context, 'api',
+            override_var=override_var)
+
+    def test_override_var_missing(self):
+        addr = '192.0.2.1'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    'api_address_family': 'ipv4',
+                    'api_interface': 'fake-interface',
+                    'ansible_facts': {
+                        'fake_interface': {
+                            'ipv4': {
+                                'address': addr,
+                            },
+                        },
+                    },
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', None, override_var))
+
 
 class TestKollaUrlFilter(unittest.TestCase):
 

releasenotes/notes/prometheus-target-address-4d2d5624ee6ae5a0.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for overriding the IP address used by Prometheus server to
+    collect metrics from exporters. This is done by specifying
+    ``prometheus_target_address`` as a host variable.