From 656f6cdb083c4b40e9c6e12863631b80b8f6d1b5 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Tue, 10 May 2022 16:36:59 +0100
Subject: [PATCH] Put openstack exporter behind HAproxy so only one is queried
at a time
Closes-Bug: #1972818
Change-Id: I9e36b9169b6725bf6db953e464fc099087747778
---
ansible/roles/prometheus/defaults/main.yml | 6 ++++++
.../roles/prometheus/templates/prometheus.yml.j2 | 4 +---
.../openstack-exporter-hammering-os-apis.yaml | 14 ++++++++++++++
3 files changed, 21 insertions(+), 3 deletions(-)
create mode 100644 releasenotes/notes/openstack-exporter-hammering-os-apis.yaml
diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml
index ad8d00b922..9ef0ef2f1f 100644
--- a/ansible/roles/prometheus/defaults/main.yml
+++ b/ansible/roles/prometheus/defaults/main.yml
@@ -79,6 +79,12 @@ prometheus_services:
image: "{{ prometheus_openstack_exporter_image_full }}"
volumes: "{{ prometheus_openstack_exporter_default_volumes + prometheus_openstack_exporter_extra_volumes }}"
dimensions: "{{ prometheus_openstack_exporter_dimensions }}"
+ haproxy:
+ prometheus_openstack_exporter:
+ enabled: "{{ enable_prometheus_openstack_exporter | bool }}"
+ mode: "http"
+ external: false
+ port: "{{ prometheus_openstack_exporter_port }}"
prometheus-elasticsearch-exporter:
container_name: prometheus_elasticsearch_exporter
group: prometheus-elasticsearch-exporter
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
index 09ab077e1c..cd36d72880 100644
--- a/ansible/roles/prometheus/templates/prometheus.yml.j2
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -100,9 +100,7 @@ scrape_configs:
honor_labels: true
static_configs:
- targets:
-{% for host in groups["prometheus-openstack-exporter"] %}
- - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_openstack_exporter_port'] }}'
-{% endfor %}
+ - '{{ kolla_internal_vip_address | put_address_in_context('url') }}:{{ prometheus_openstack_exporter_port }}'
{% endif %}
{% if enable_prometheus_elasticsearch_exporter | bool %}
diff --git a/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml b/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml
new file mode 100644
index 0000000000..78ef561c9d
--- /dev/null
+++ b/releasenotes/notes/openstack-exporter-hammering-os-apis.yaml
@@ -0,0 +1,14 @@
+---
+fixes:
+ - |
+ The prometheus openstack exporters are now behind haproxy,
+ providing a unique time series in the prometheus database.
+ Also ensures that only one exporter queries
+ the openstack APIs at any given time interval.
+ With the previous behavior each openstack exporter
+ was scraped at the same time.
+ This caused each exporter to query the openstack APIs
+ simultaneously introducing unneccesary load and duplicate
+ time series in the prometheus database due to the instance
+ label being unique for each exporter.
+ `LP#1972818 <https://bugs.launchpad.net/kolla-ansible/+bug/1972818>`__