From 4ff65b7661ea06e9fa8631c4eb82232e03af77d7 Mon Sep 17 00:00:00 2001
From: Piotr Parczewski <piotr@stackhpc.com>
Date: Mon, 20 Sep 2021 15:45:11 +0200
Subject: [PATCH] Use friendly target names in Prometheus
Change-Id: I16fdb2f93ddb656eeacd3f2b84190f9bdcfaa21c
---
.../prometheus/templates/prometheus.yml.j2 | 72 +++++++++++++++----
...metheus-target-names-94b17f58fa7c957a.yaml | 13 ++++
2 files changed, 73 insertions(+), 12 deletions(-)
create mode 100644 releasenotes/notes/prometheus-target-names-94b17f58fa7c957a.yaml
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
index ac11aa3869..1e829648ac 100644
--- a/ansible/roles/prometheus/templates/prometheus.yml.j2
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -17,16 +17,24 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['prometheus'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}'
+ - '{{ host }}:{{ prometheus_port }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% if enable_prometheus_node_exporter | bool %}
- job_name: node
static_configs:
- targets:
{% for host in groups['prometheus-node-exporter'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_mysqld_exporter | bool %}
@@ -34,8 +42,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['prometheus-mysqld-exporter'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_haproxy_exporter | bool %}
@@ -43,8 +55,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['prometheus-haproxy-exporter'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_rabbitmq_exporter | bool %}
@@ -52,8 +68,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['rabbitmq'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_memcached_exporter | bool %}
@@ -61,8 +81,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['prometheus-memcached-exporter'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_cadvisor | bool %}
@@ -70,8 +94,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups["prometheus-cadvisor"] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_fluentd_integration | bool %}
@@ -79,8 +107,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['fluentd'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_ceph_mgr_exporter | bool %}
@@ -91,6 +123,10 @@ scrape_configs:
{% for exporter in prometheus_ceph_mgr_exporter_endpoints %}
- '{{ exporter }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_openstack_exporter | bool %}
@@ -101,8 +137,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups["prometheus-openstack-exporter"] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_openstack_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_openstack_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_elasticsearch_exporter | bool %}
@@ -111,8 +151,12 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups["prometheus-elasticsearch-exporter"] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
{% endif %}
{% if enable_prometheus_blackbox_exporter | bool and prometheus_blackbox_exporter_endpoints | length > 0 | bool %}
@@ -150,14 +194,18 @@ scrape_configs:
static_configs:
- targets:
{% for host in groups['prometheus-alertmanager'] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
{% endfor %}
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: "([^:]+):\\d+"
+ target_label: instance
alerting:
alertmanagers:
- static_configs:
- targets:
{% for host in groups["prometheus-alertmanager"] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+ - '{{ host }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
{% endfor %}
{% endif %}
diff --git a/releasenotes/notes/prometheus-target-names-94b17f58fa7c957a.yaml b/releasenotes/notes/prometheus-target-names-94b17f58fa7c957a.yaml
new file mode 100644
index 0000000000..c06fb470ea
--- /dev/null
+++ b/releasenotes/notes/prometheus-target-names-94b17f58fa7c957a.yaml
@@ -0,0 +1,13 @@
+---
+features:
+ - |
+ Changes target names in Prometheus to user-friendly, Ansible inventory
+ based values.
+upgrade:
+ - |
+ Changes Prometheus targets naming. This makes their names more user
+ friendly but also creates a completely new set of a time series data.
+ New target names are taken from Ansible inventory and have the exporter
+ port number stripped off. Any Grafana dashboard that relies on a specific,
+ hard-coded naming pattern for the targets will stop showing metrics after
+ the upgrade.