Enable HAProxy Prometheus metrics endpoint

HAProxy exposes a Prometheus metrics endpoint, it just needs to be
enabled. Enable this and remove configuration for
prometheus-haproxy-exporter. Remaining prometheus-haproxy-exporter
containers will automatically be removed.

Change-Id: If6e75691d2a996b06a9b95cb0aae772db54389fb
Co-Authored-By: Matt Anson <matta@stackhpc.com>

ansible/inventory/all-in-one

@@ -620,9 +620,6 @@ storage
 [prometheus-mysqld-exporter:children]
 mariadb
 
-[prometheus-haproxy-exporter:children]
-loadbalancer
-
 [prometheus-memcached-exporter:children]
 memcached
 

ansible/inventory/multinode

@@ -639,9 +639,6 @@ storage
 [prometheus-mysqld-exporter:children]
 mariadb
 
-[prometheus-haproxy-exporter:children]
-loadbalancer
-
 [prometheus-memcached-exporter:children]
 memcached
 

ansible/roles/loadbalancer/tasks/upgrade.yml

@@ -1,2 +1,20 @@
 ---
+# TODO(dawudm) We should replace these two tasks with a task to check the port
+# is free in precheck.yml in the D release
+- name: Stop and remove containers for haproxy exporter containers
+  become: true
+  kolla_container:
+    action: "stop_and_remove_container"
+    name: "prometheus_haproxy_exporter"
+  when:
+    - inventory_hostname in groups['loadbalancer']
+
+- name: Removing config for haproxy exporter
+  file:
+    path: "{{ node_config_directory }}/prometheus-haproxy-exporter"
+    state: "absent"
+  become: true
+  when:
+    - inventory_hostname in groups['loadbalancer']
+
 - import_tasks: deploy.yml

ansible/roles/loadbalancer/templates/haproxy/haproxy_main.cfg.j2

@@ -42,6 +42,13 @@ listen stats
    stats realm Haproxy\ Stats
    stats auth {{ haproxy_user }}:{{ haproxy_password }}
 
+{% if enable_prometheus_haproxy_exporter | bool %}
+listen metrics
+   bind {{ api_interface_address }}:{{ prometheus_haproxy_exporter_port }}
+   mode http
+   http-request use-service prometheus-exporter if { path /metrics }
+{% endif %}
+
 frontend status
     bind {{ api_interface_address }}:{{ haproxy_monitor_port }}
     {% if api_interface_address != kolla_internal_vip_address %}

ansible/roles/prometheus/defaults/main.yml

@@ -37,13 +37,6 @@ prometheus_services:
     image: "{{ prometheus_mysqld_exporter_image_full }}"
     volumes: "{{ prometheus_mysqld_exporter_default_volumes + prometheus_mysqld_exporter_extra_volumes }}"
     dimensions: "{{ prometheus_mysqld_exporter_dimensions }}"
-  prometheus-haproxy-exporter:
-    container_name: prometheus_haproxy_exporter
-    group: prometheus-haproxy-exporter
-    enabled: "{{ enable_prometheus_haproxy_exporter | bool }}"
-    image: "{{ prometheus_haproxy_exporter_image_full }}"
-    volumes: "{{ prometheus_haproxy_exporter_default_volumes + prometheus_haproxy_exporter_extra_volumes }}"
-    dimensions: "{{ prometheus_haproxy_exporter_dimensions }}"
   prometheus-memcached-exporter:
     container_name: prometheus_memcached_exporter
     group: prometheus-memcached-exporter
@@ -197,10 +190,6 @@ prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}
 prometheus_server_tag: "{{ prometheus_tag }}"
 prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}"
 
-prometheus_haproxy_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-haproxy-exporter"
-prometheus_haproxy_exporter_tag: "{{ prometheus_tag }}"
-prometheus_haproxy_exporter_image_full: "{{ prometheus_haproxy_exporter_image }}:{{ prometheus_haproxy_exporter_tag }}"
-
 prometheus_mysqld_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/prometheus-mysqld-exporter"
 prometheus_mysqld_exporter_tag: "{{ prometheus_tag }}"
 prometheus_mysqld_exporter_image_full: "{{ prometheus_mysqld_exporter_image }}:{{ prometheus_mysqld_exporter_tag }}"
@@ -243,7 +232,6 @@ prometheus_msteams_tag: "{{ prometheus_tag }}"
 prometheus_msteams_image_full: "{{ prometheus_msteams_image }}:{{ prometheus_msteams_tag }}"
 
 prometheus_server_dimensions: "{{ default_container_dimensions }}"
-prometheus_haproxy_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_mysqld_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_node_exporter_dimensions: "{{ default_container_dimensions }}"
 prometheus_memcached_exporter_dimensions: "{{ default_container_dimensions }}"
@@ -261,12 +249,6 @@ prometheus_server_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "prometheus_v2:/var/lib/prometheus"
   - "kolla_logs:/var/log/kolla/"
-prometheus_haproxy_exporter_default_volumes:
-  - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro"
-  - "/etc/localtime:/etc/localtime:ro"
-  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "kolla_logs:/var/log/kolla/"
-  - "haproxy_socket:/var/lib/kolla/haproxy"
 prometheus_mysqld_exporter_default_volumes:
   - "{{ node_config_directory }}/prometheus-mysqld-exporter/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
@@ -327,7 +309,6 @@ prometheus_msteams_default_volumes:
 
 prometheus_extra_volumes: "{{ default_extra_volumes }}"
 prometheus_server_extra_volumes: "{{ prometheus_extra_volumes }}"
-prometheus_haproxy_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_mysqld_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_node_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
 prometheus_memcached_exporter_extra_volumes: "{{ prometheus_extra_volumes }}"
@@ -356,7 +337,6 @@ prometheus_server_command: >-
 prometheus_blackbox_exporter_cmdline_extras: ""
 prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process"
 prometheus_elasticsearch_exporter_cmdline_extras: ""
-prometheus_haproxy_exporter_cmdline_extras: ""
 prometheus_memcached_exporter_cmdline_extras: ""
 prometheus_mysqld_exporter_cmdline_extras: ""
 prometheus_node_exporter_cmdline_extras: ""

ansible/roles/prometheus/handlers/main.yml

@@ -45,21 +45,6 @@
   when:
     - kolla_action != "config"
 
-- name: Restart prometheus-haproxy-exporter container
-  vars:
-    service_name: "prometheus-haproxy-exporter"
-    service: "{{ prometheus_services[service_name] }}"
-  become: true
-  kolla_container:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-
 - name: Restart prometheus-memcached-exporter container
   vars:
     service_name: "prometheus-memcached-exporter"

ansible/roles/prometheus/tasks/precheck.yml

@@ -12,7 +12,6 @@
     name:
       - prometheus_server
       - prometheus_node_exporter
-      - prometheus_haproxy_exporter
       - prometheus_memcached_exporter
       - prometheus_mysqld_exporter
       - prometheus_cadvisor
@@ -83,18 +82,6 @@
     - inventory_hostname in groups['prometheus-mysqld-exporter']
     - enable_prometheus_mysqld_exporter | bool
 
-- name: Checking free port for Prometheus haproxy_exporter
-  wait_for:
-    host: "{{ 'api' | kolla_address }}"
-    port: "{{ prometheus_haproxy_exporter_port }}"
-    connect_timeout: 1
-    timeout: 1
-    state: stopped
-  when:
-    - container_facts['prometheus_haproxy_exporter'] is not defined
-    - inventory_hostname in groups['prometheus-haproxy-exporter']
-    - enable_prometheus_haproxy_exporter | bool
-
 - name: Checking free port for Prometheus memcached_exporter
   wait_for:
     host: "{{ 'api' | kolla_address }}"

ansible/roles/prometheus/templates/prometheus-haproxy-exporter.json.j2

@@ -1,11 +0,0 @@
-{
-    "command": "/opt/haproxy_exporter/haproxy_exporter --haproxy.scrape-uri unix:/var/lib/kolla/haproxy/haproxy.sock --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}{% if prometheus_haproxy_exporter_cmdline_extras %} {{ prometheus_haproxy_exporter_cmdline_extras }}{% endif %}",
-    "config_files": [],
-    "permissions": [
-        {
-            "path": "/var/log/kolla/prometheus",
-            "owner": "prometheus:kolla",
-            "recurse": true
-        }
-    ]
-}

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -60,9 +60,9 @@ scrape_configs:
 {% if enable_prometheus_haproxy_exporter | bool %}
   - job_name: haproxy
     static_configs:
-{% for host in groups['prometheus-haproxy-exporter'] %}
+{% for host in groups['loadbalancer'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"

ansible/site.yml

@@ -452,7 +452,6 @@
     - prometheus
     - prometheus-node-exporter
     - prometheus-mysqld-exporter
-    - prometheus-haproxy-exporter
     - prometheus-memcached-exporter
     - prometheus-cadvisor
     - prometheus-alertmanager

etc/kolla/globals.yml

@@ -792,7 +792,6 @@ workaround_ansible_issue_8743: yes
 # Extra parameters passed to Prometheus exporters.
 #prometheus_blackbox_exporter_cmdline_extras:
 #prometheus_elasticsearch_exporter_cmdline_extras:
-#prometheus_haproxy_exporter_cmdline_extras:
 #prometheus_memcached_exporter_cmdline_extras:
 #prometheus_mysqld_exporter_cmdline_extras:
 #prometheus_node_exporter_cmdline_extras:

releasenotes/notes/haproxy-metrics-endpoint-d62c0f34e29a10b2.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    Removed configuration and deployment of ``prometheus-haproxy-exporter`` as
+    its repository is now archived. We now use the native support for
+    Prometheus which is now built into HAProxy. For consistency this is exposed
+    on the ``prometheus_haproxy_exporter_port`` port.
+    ``prometheus-haproxy-exporter`` containers and config are automatically
+    removed.
+
+upgrade:
+  - |
+    Due to the change from using the ``prometheus-haproxy-exporter`` to using
+    the native support for Prometheus which is now built into HAProxy, metric
+    names may have been replaced and/or removed, and in some cases the metric
+    names may have remained the same but the labels may have changed. Alerts
+    and dashboards may also need to be updated to use the new metrics. Please
+    review any configuration that references the old metrics as this is not a
+    backwards compatible change.

specs/prometheus.rst

@@ -261,9 +261,9 @@ As with all optional services in Kolla Ansible, Prometheus deployment should be
 controlled by Kolla Ansible variables.  A high level enable_prometheus variable
 should control whether Prometheus is used at all.  Additionally, additional
 variables can be used to control individual exporters.  For example,
-enable_prometheus_haproxy could be used to enable/disable the HAProxy exporter
+enable_prometheus_haproxy_exporter could be used to enable/disable the HAProxy
-to Prometheus.  By default Prometheus should be enabled and exporters should be
+exporter to Prometheus.  By default Prometheus should be enabled and exporters
-enabled if both Prometheus and the associated service are enabled.
+should be enabled if both Prometheus and the associated service are enabled.
 
 Limitations
 -----------

tests/templates/inventory.j2

@@ -681,9 +681,6 @@ storage
 [prometheus-mysqld-exporter:children]
 mariadb
 
-[prometheus-haproxy-exporter:children]
-loadbalancer
-
 [prometheus-memcached-exporter:children]
 memcached