From 0ec41f2092dd806839fe5a1c17c2d0495dd98e44 Mon Sep 17 00:00:00 2001
From: Jorge Niedbalski <jorge.niedbalski@linaro.org>
Date: Tue, 12 Jun 2018 12:58:51 -0400
Subject: [PATCH] [prometheus] Allow custom alert rules to be configured.

This patch extends the configuration task for prometheus
to allow the operator to pass a(set) of prometheus alert
rules files, that will be used by alertmanager to produce
alerts.

This functionality is only enabled when the prometheus-alertmanager
service is enabled.

Change-Id: I882759c3774f43640631c1058f8a9cb24e7a60d2
Closes-Bug: #1776529
Signed-off-by: Jorge Niedbalski <jorge.niedbalski@linaro.org>
---
 ansible/roles/prometheus/handlers/main.yml    |  1 +
 ansible/roles/prometheus/tasks/config.yml     | 40 ++++++++++++++++---
 .../templates/prometheus-server.json.j2       |  9 +++++
 .../prometheus/templates/prometheus.yml.j2    |  7 ++++
 4 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml
index bcac309d5b..0538edfd46 100644
--- a/ansible/roles/prometheus/handlers/main.yml
+++ b/ansible/roles/prometheus/handlers/main.yml
@@ -138,4 +138,5 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - config_json.changed | bool
+      or prometheus_alertmanager_confs.changed | bool
       or prometheus_container.changed | bool
diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml
index a1b075b870..4acf1d6eb8 100644
--- a/ansible/roles/prometheus/tasks/config.yml
+++ b/ansible/roles/prometheus/tasks/config.yml
@@ -24,6 +24,32 @@
   notify:
     - Restart {{ item.key }} container
 
+- name: Find custom prometheus alert rules files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/prometheus/"
+    pattern: "*.rules"
+  run_once: True
+  register: prometheus_alert_rules
+  when:
+    - enable_prometheus_alertmanager | bool
+
+- name: Copying over custom prometheus alert rules files
+  vars:
+    service: "{{ prometheus_services['prometheus-server']}}"
+  copy:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/prometheus-server/{{ item.path | basename }}"
+    mode: "0660"
+  register: prometheus_confs
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool and enable_prometheus_alertmanager | bool
+    - prometheus_alert_rules is defined and prometheus_alert_rules.files | length > 0
+  with_items: "{{ prometheus_alert_rules.files }}"
+  notify:
+    - Restart prometheus-server container
+
 - name: Copying over prometheus config file
   vars:
     service: "{{ prometheus_services['prometheus-server']}}"
@@ -44,16 +70,17 @@
 - name: Copying over prometheus alertmanager config file
   vars:
     service: "{{ prometheus_services['prometheus-alertmanager']}}"
-  merge_yaml:
-    sources:
-      - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-alertmanager.yml"
-      - "{{ node_custom_config }}/prometheus/prometheus-alertmanager.yml"
-      - "{{ role_path }}/templates/prometheus-alertmanager.yml.j2"
+  template:
+    src: "{{ item }}"
     dest: "{{ node_config_directory }}/prometheus-alertmanager/prometheus-alertmanager.yml"
-  register: prometheus_confs
+  register: prometheus_alertmanager_confs
   when:
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
+  with_first_found:
+    - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus-alertmanager.yml"
+    - "{{ node_custom_config }}/prometheus/prometheus-alertmanager.yml"
+    - "{{ role_path }}/templates/prometheus-alertmanager.yml.j2"
   notify:
     - Restart prometheus-alertmanager container
 
@@ -81,6 +108,7 @@
     name: "{{ item.value.container_name }}"
     image: "{{ item.value.image }}"
     volumes: "{{ item.value.volumes }}"
+    dimensions: "{{ item.value.dimensions }}"
   register: check_prometheus_containers
   when:
     - kolla_action != "config"
diff --git a/ansible/roles/prometheus/templates/prometheus-server.json.j2 b/ansible/roles/prometheus/templates/prometheus-server.json.j2
index 46b70425cd..3470e6de93 100644
--- a/ansible/roles/prometheus/templates/prometheus-server.json.j2
+++ b/ansible/roles/prometheus/templates/prometheus-server.json.j2
@@ -7,6 +7,15 @@
             "owner": "prometheus",
             "perm": "0600"
         }
+{% if enable_prometheus_alertmanager %}
+        ,{
+            "source": "{{ container_config_directory }}/*.rules",
+            "dest": "/etc/prometheus/",
+            "optional": true,
+            "owner": "prometheus",
+            "perm": "0600"
+        }
+{% endif %}
     ],
     "permissions": [
         {
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
index 6de1ca9ae7..5bcb48dbf6 100644
--- a/ansible/roles/prometheus/templates/prometheus.yml.j2
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -5,6 +5,13 @@ global:
   external_labels:
     monitor: 'kolla'
 
+{% if prometheus_alert_rules is defined and prometheus_alert_rules.files | length  > 0 %}
+rule_files:
+{% for rule in prometheus_alert_rules.files %}
+  - "/etc/prometheus/{{ rule.path | basename }}"
+{% endfor %}
+{% endif %}
+
 scrape_configs:
   - job_name: prometheus
     static_configs: