Add prometheus annotations to calico-nodes for felix metrics

This adds the prometheus annotations to the calico-node daemonset
to allow prometheus to create a scrape config for calico metrics.
This requires adding a annotation tree in the chart's values.yaml
file

Change-Id: I0e62fce34ea8de6d0241ea00aaae66187b808c81

calico/templates/daemonset-calico-node.yaml

@@ -51,6 +51,7 @@ spec:
         # reserves resources for critical add-on pods so that they can be rescheduled after
         # a failure.  This annotation works in tandem with the toleration below.
         scheduler.alpha.kubernetes.io/critical-pod: ''
+{{ tuple $envAll.Values.pod.annotations.calico_node | include "helm-toolkit.snippets.prometheus_pod_annotations" | indent 8 }}
     spec:
       hostNetwork: true
       tolerations:

calico/values.yaml

@@ -44,6 +44,10 @@ images:
       - calico_kube_policy_controller
 
 pod:
+  annotations:
+    calico_node:
+      prometheus_port: 9091
+      prometheus_scrape: true
   resources:
     enabled: false
     jobs:

helm-toolkit/templates/snippets/_prometheus_pod_annotations.tpl

@@ -23,8 +23,13 @@ limitations under the License.
 
 {{- define "helm-toolkit.snippets.prometheus_pod_annotations" -}}
 {{- $pod := index . 0 -}}
-{{- $context := index . 1 -}}
+{{- if $pod.prometheus_scrape }}
-prometheus.io/scrape: {{ $pod.scrape | quote }}
+prometheus.io/scrape: {{ $pod.prometheus_scrape | quote }}
-prometheus.io/path: {{ $pod.path.default | quote }}
+{{- end }}
-prometheus.io/port: {{ $pod.scrape_port | quote }}
+{{- if $pod.prometheus_path }}
+prometheus.io/path: {{ $pod.prometheus_path | quote }}
+{{- end }}
+{{- if $pod.prometheus_port }}
+prometheus.io/port: {{ $pod.prometheus_port | quote }}
+{{- end }}
 {{- end -}}

prometheus/values.yaml

@@ -387,6 +387,39 @@ conf:
               - __meta_kubernetes_service_name
             target_label: job
             replacement: ${1}
+        # Example scrape config for pods
+        #
+        # The relabeling allows the actual pod scrape endpoint to be configured via the
+        # following annotations:
+        #
+        # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+        # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+        # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
+        # pod's declared ports (default is a port-free target if none are declared).
+        - job_name: 'kubernetes-pods'
+          kubernetes_sd_configs:
+          - role: pod
+          relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: true
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+            action: replace
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+            target_label: __address__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - source_labels: [__meta_kubernetes_namespace]
+            action: replace
+            target_label: kubernetes_namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            action: replace
+            target_label: kubernetes_pod_name
         - job_name: calico-etcd
           honor_labels: false
           kubernetes_sd_configs: