prometheus exporter to monitor health check for web URL

1) A separate prometheus job need to provide target and scrap metrics 2) it is based on https://github.com/prometheus/blackbox_exporter Adding script file for deployment and job under zuul.d Resolving conflict Change-Id: Ia15ab7d8ef882886fe0e37cc2599e6815d7bcc6c
2020-05-13 21:23:10 +05:30 · 2020-05-13 21:23:10 +05:30 · 628186f05e
commit 628186f05e
parent 817efe966c
9 changed files with 287 additions and 0 deletions
--- a/prometheus-blackbox-exporter/Chart.yaml
+++ b/prometheus-blackbox-exporter/Chart.yaml
@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: v1
+description: OpenStack-Helm blackbox exporter for Prometheus
+name: prometheus-blackbox-exporter
+version: 0.1.0
+sources:
+  - https://opendev.org/openstack/openstack-helm-infra
+  - https://github.com/prometheus/blackbox_exporter
+maintainers:
+  - name: OpenStack-Helm Authors
+...
--- a/prometheus-blackbox-exporter/requirements.yaml
+++ b/prometheus-blackbox-exporter/requirements.yaml
@ -0,0 +1,6 @@
+---
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0
+...
--- a/prometheus-blackbox-exporter/templates/deployment.yaml
+++ b/prometheus-blackbox-exporter/templates/deployment.yaml
@ -0,0 +1,67 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- $envAll := . }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-blackbox-exporter
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 2 }}
+spec:
+  replicas: {{ .Values.pod.replicas.prometheus_blackbox_exporter }}
+  selector:
+    matchLabels:
+{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ dict "envAll" $envAll "podName" "prometheus-blackbox-exporter" "containerNames" (list "blackbox-exporter") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
+    spec:
+      nodeSelector:
+        {{ .Values.labels.blackbox_exporter.node_selector_key }}: {{ .Values.labels.blackbox_exporter.node_selector_value | quote }}
+      containers:
+      - name: blackbox-exporter
+{{ tuple $envAll "prometheus_blackbox_exporter" | include "helm-toolkit.snippets.image" | indent 8 }}
+{{ tuple $envAll $envAll.Values.pod.resources.prometheus_blackbox_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
+        args:
+          - "--config.file=/config/blackbox.yaml"
+        ports:
+          - name: metrics
+            containerPort: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          initialDelaySeconds: 20
+          periodSeconds: 30
+        volumeMounts:
+        - mountPath: /config/blackbox.yaml
+          name: config
+          subPath: blackbox.yaml
+      volumes:
+        - name: config
+          secret:
+            secretName: prometheus-blackbox-exporter-etc
--- a/prometheus-blackbox-exporter/templates/secret.yaml
+++ b/prometheus-blackbox-exporter/templates/secret.yaml
@ -0,0 +1,23 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- $envAll := . }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus-blackbox-exporter-etc
+  labels:
+{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+data:
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.config.blackbox "key" "blackbox.yaml" "format" "Secret") | indent 2 }}
--- a/prometheus-blackbox-exporter/templates/service.yaml
+++ b/prometheus-blackbox-exporter/templates/service.yaml
@ -0,0 +1,26 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- $envAll := . }}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "prometheus_blackbox_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+    - name: metrics
+      port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  selector:
+{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
--- a/prometheus-blackbox-exporter/values.yaml
+++ b/prometheus-blackbox-exporter/values.yaml
@ -0,0 +1,113 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for kube-state-metrics.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+---
+
+images:
+  tags:
+    prometheus_blackbox_exporter: docker.io/prom/blackbox-exporter:v0.16.0
+  pull_policy: IfNotPresent
+  local_registry:
+    active: false
+labels:
+  blackbox_exporter:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+service:
+  annotations: {}
+  port: 9115
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  prometheus_blackbox_exporter:
+    namespace: null
+    hosts:
+      default: prometheus-blackbox-exporter
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: 'http'
+    port:
+      metrics:
+        default: 9115
+
+pod:
+  replicas:
+    prometheus_blackbox_exporter: 1
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: "9115"
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+
+  lifecycle:
+    upgrades:
+      deployments:
+        revision_history: 3
+        pod_replacement_strategy: RollingUpdate
+        rolling_update:
+          max_unavailable: 1
+          max_surge: 3
+    termination_grace_period:
+      prometheus_blackbox_exporter:
+        timeout: 30
+  resources:
+    enabled: true
+    prometheus_blackbox_exporter:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - prometheus-openstack-exporter-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+    prometheus_blackbox_exporter:
+      jobs:
+        - prometheus-openstack-exporter-ks-user
+      services:
+        - endpoint: internal
+          service: identity
+
+config:
+  blackbox:
+    modules:
+      http_2xx:
+        prober: http
+        timeout: 10s
+        http:
+          valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+          no_follow_redirects: false
+          preferred_ip_protocol: "ip4"
+...
--- a/tools/deployment/common/blackbox-exporter.sh
+++ b/tools/deployment/common/blackbox-exporter.sh
@ -0,0 +1,28 @@
+#!/bin/bash
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make prometheus-blackbox-exporter
+
+#NOTE: Deploy command
+helm upgrade --install prometheus-blackbox-exporter \
+    ./prometheus-blackbox-exporter --namespace=osh-infra
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status prometheus-blackbox-exporter
--- a/tools/deployment/osh-infra-monitoring/105-blackbox-exporter.sh
+++ b/tools/deployment/osh-infra-monitoring/105-blackbox-exporter.sh
@ -0,0 +1 @@
+../common/blackbox-exporter.sh
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -195,6 +195,7 @@
          - ./tools/deployment/osh-infra-monitoring/080-node-exporter.sh
        - - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
          - ./tools/deployment/osh-infra-monitoring/100-openstack-exporter.sh
+          - ./tools/deployment/osh-infra-monitoring/105-blackbox-exporter.sh
          - ./tools/deployment/osh-infra-monitoring/110-grafana.sh
          - ./tools/deployment/osh-infra-monitoring/120-nagios.sh
          - ./tools/deployment/osh-infra-monitoring/130-postgresql.sh