openstack/openstack-helm-infra
Code Issues Proposed changes

Merge "Fluentd: Switch to Native Metrics Plugin"

Browse Source
This commit is contained in:
Zuul 2020-04-02 20:50:53 +00:00 committed by Gerrit Code Review
commit e42a628243
12 changed files with 56 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fluentd/templates/deployment-fluentd.yaml
View File

@ -18,6 +18,7 @@ limitations under the License.
{{- $envAll := . }}
{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }}
{{- $kafkaBroker := tuple "kafka" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
@ -100,6 +101,9 @@ spec:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
{{- if .Values.monitoring.prometheus.enabled }}
{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_pod_annotations" | indent 8 }}
{{- end }}
{{ dict "envAll" $envAll "podName" "fluentd" "containerNames" (list "fluentd" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}

30
fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl
View File

@ -1,30 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
COMMAND="${@:-start}"
function start () {
exec fluentd_exporter --scrape_uri "$FLUENTD_METRICS_HOST"
}
function stop () {
kill -TERM 1
}
$COMMAND

27
fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml
View File

@ -1,27 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.configmap_bin_exporter .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-exporter-bin" }}
data:
fluentd-exporter.sh: |
{{ tuple "bin/_fluentd-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

85
fluentd/templates/monitoring/prometheus/exporter-deployment.yaml
View File

@ -1,85 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
{{ $fluentd_host := tuple "fluentd" "internal" "metrics" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{ $fluentd_metrics_path := "api/plugins.json" }}
{{ $fluentd_metrics_host := printf "http://%s/%s" $fluentd_host $fluentd_metrics_path }}
{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd-exporter" }}
{{ tuple $envAll "prometheus_fluentd_exporter" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $rcControllerName | quote }}
labels:
{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.prometheus_fluentd_exporter }}
selector:
matchLabels:
{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $rcControllerName | quote }}
nodeSelector:
{{ .Values.labels.prometheus_fluentd_exporter.node_selector_key }}: {{ .Values.labels.prometheus_fluentd_exporter.node_selector_value | quote }}
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_fluentd_exporter.timeout | default "30" }}
initContainers:
{{ tuple $envAll "prometheus_fluentd_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: fluentd-exporter
{{ tuple $envAll "prometheus_fluentd_exporter" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.prometheus_fluentd_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "exporter" "container" "fluentd_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/fluentd-exporter.sh
- start
ports:
- name: metrics
containerPort: {{ tuple "prometheus_fluentd_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
readinessProbe:
tcpSocket:
port: {{ tuple "prometheus_fluentd_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
initialDelaySeconds: 20
periodSeconds: 10
env:
- name: FLUENTD_METRICS_HOST
value: {{ $fluentd_metrics_host }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: fluentd-exporter-bin
mountPath: /tmp/fluentd-exporter.sh
subPath: fluentd-exporter.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: fluentd-exporter-bin
configMap:
name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-exporter-bin" | quote }}
defaultMode: 0555
{{- end }}

20
fluentd/templates/monitoring/prometheus/exporter-network-policy.yaml
View File

@ -1,20 +0,0 @@
{{/*
Copyright 2019 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" -}}
{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
{{- end -}}

37
fluentd/templates/monitoring/prometheus/exporter-service.yaml
View File

@ -1,37 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd_exporter }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "prometheus_fluentd_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
labels:
{{ tuple $envAll "prometheus-fluentd-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
{{- if .Values.monitoring.prometheus.enabled }}
{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
{{- end }}
spec:
ports:
- name: metrics
port: {{ tuple "prometheus_fluentd_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{- end }}

2
fluentd/templates/service-fluentd.yaml
View File

@ -28,8 +28,6 @@ spec:
{{ if .Values.network.fluentd.node_port.enabled }}
nodePort: {{ .Values.network.fluentd.node_port.port }}
{{ end }}
- name: metrics
port: {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ if .Values.network.fluentd.node_port.enabled }}

80
fluentd/values.yaml
View File

@ -25,14 +25,10 @@ labels:
fluentd:
node_selector_key: openstack-control-plane
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
images:
tags:
fluentd: docker.io/openstackhelm/fluentd:debian-20190903
prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0
fluentd: docker.io/openstackhelm/fluentd:debian-20200324
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
elasticsearch_template: docker.io/openstackhelm/heat:newton-ubuntu_xenial
@ -60,18 +56,25 @@ dependencies:
services:
- endpoint: internal
service: local_image_registry
prometheus_fluentd_exporter:
services:
- endpoint: internal
service: fluentd
conf:
fluentd:
template: |
<source>
bind 0.0.0.0
port 24220
@type monitor_agent
@type prometheus
port 24231
</source>
<source>
@type prometheus_monitor
</source>
<source>
@type prometheus_output_monitor
</source>
<source>
@type prometheus_tail_monitor
</source>
<source>
@ -291,10 +294,6 @@ conf:
type_name fluent
user "#{ENV['ELASTICSEARCH_USERNAME']}"
</match>
fluentd_exporter:
log:
format: "logger:stdout?json=true"
level: "info"
endpoints:
cluster_domain_suffix: cluster.local
@ -346,7 +345,7 @@ endpoints:
service:
default: 24224
metrics:
default: 24220
default: 24231
kafka:
namespace: null
name: kafka
@ -367,25 +366,13 @@ endpoints:
broker:
default: 9092
public: 80
prometheus_fluentd_exporter:
namespace: null
hosts:
default: fluentd-exporter
host_fqdn_override:
default: null
path:
default: /metrics
scheme:
default: 'http'
port:
metrics:
default: 9309
monitoring:
prometheus:
enabled: false
fluentd_exporter:
enabled: true
fluentd:
scrape: true
port: 24231
network:
fluentd:
@ -394,11 +381,6 @@ network:
port: 32329
network_policy:
prometheus-fluentd-exporter:
ingress:
- {}
egress:
- {}
fluentd:
ingress:
- {}
@ -421,13 +403,7 @@ pod:
fluentd:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
exporter:
pod:
runAsUser: 65534
container:
fluentd_exporter:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
affinity:
anti:
type:
@ -453,11 +429,8 @@ pod:
termination_grace_period:
fluentd:
timeout: 30
prometheus_fluentd_exporter:
timeout: 30
replicas:
fluentd: 3
prometheus_fluentd_exporter: 1
resources:
enabled: false
fluentd:
@ -467,13 +440,6 @@ pod:
requests:
memory: '128Mi'
cpu: '500m'
prometheus_fluentd_exporter:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "128Mi"
cpu: "500m"
mounts:
fluentd:
fluentd:
@ -483,12 +449,6 @@ manifests:
configmap_etc: true
deployment_fluentd: true
job_image_repo_sync: true
monitoring:
prometheus:
configmap_bin_exporter: true
deployment_exporter: true
network_policy_exporter: false
service_exporter: true
network_policy: false
secret_elasticsearch: true
secret_fluentd_env: true

19
roles/gather-prom-metrics/tasks/main.yaml
View File

@ -15,7 +15,7 @@
path: "{{ logs_dir }}/prometheus"
state: directory
- name: "Get prometheus metrics from exporters in all namespaces"
- name: "Get metrics from exporter services in all namespaces"
shell: |-
set -e
NAMESPACES=$(kubectl get namespaces -o json | jq -r '.items[].metadata.name')
@ -23,6 +23,7 @@
SERVICES=$(kubectl get svc -n $NS -o json | jq -r '.items[] | select(.spec.ports[].name=="metrics") | .metadata.name')
for SVC in $SERVICES; do
PORT=$(kubectl get svc $SVC -n $NS -o json | jq -r '.spec.ports[] | select(.name=="metrics") | .port')
echo "Scraping $SVC.$NS:$PORT/metrics:"
curl "$SVC.$NS:$PORT/metrics" >> "{{ logs_dir }}"/prometheus/$NS-$SVC.txt || true
done
done
@ -58,6 +59,22 @@
executable: /bin/bash
ignore_errors: True
- name: "Get metrics from fluentd pods"
shell: |-
set -e
NAMESPACE="osh-infra"
APP_LABEL="fluentd"
PODS=$(kubectl get pods -n $NAMESPACE -l application=$APP_LABEL -o json | jq -r '.items[].metadata.name')
for POD in $PODS; do
IP=$(kubectl get pod -n $NAMESPACE $POD -o json | jq -r '.status.podIP')
PORT=$(kubectl get pod -n $NAMESPACE $POD -o json | jq -r '.spec.containers[0].ports[] | select(.name=="metrics") | .containerPort')
echo "Scraping $POD at $IP:$PORT/metrics"
curl "$IP:$PORT/metrics" >> "{{ logs_dir }}"/prometheus/$POD.txt || true
done
args:
executable: /bin/bash
ignore_errors: True
- name: "Downloads logs to executor"
synchronize:
src: "{{ logs_dir }}/prometheus"

3
tools/deployment/armada/manifests/armada-lma.yaml
View File

@ -524,9 +524,6 @@ data:
fluentd:
node_selector_key: openstack-control-plane
node_selector_value: enabled
prometheus_fluentd_exporter:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled

23
tools/deployment/common/fluentd-daemonset.sh
View File

@ -25,12 +25,6 @@ endpoints:
fluentd:
hosts:
default: fluentd-daemonset
prometheus_fluentd_exporter:
hosts:
default: fluentd-daemonset-exporter
monitoring:
prometheus:
enabled: true
pod:
env:
fluentd:
@ -48,9 +42,20 @@ conf:
fluentd:
template: |
<source>
bind 0.0.0.0
port 24220
@type monitor_agent
@type prometheus
port 24231
</source>
<source>
@type prometheus_monitor
</source>
<source>
@type prometheus_output_monitor
</source>
<source>
@type prometheus_tail_monitor
</source>
<source>

7
tools/deployment/common/fluentd-deployment.sh
View File

@ -25,9 +25,6 @@ if [ ! -d "/var/log/journal" ]; then
tee /tmp/fluentd.yaml << EOF
deployment:
type: Deployment
monitoring:
prometheus:
enabled: true
pod:
replicas:
fluentd: 1
@ -52,9 +49,6 @@ else
tee /tmp/fluentd.yaml << EOF
deployment:
type: Deployment
monitoring:
prometheus:
enabled: true
pod:
replicas:
fluentd: 1
@ -66,7 +60,6 @@ helm upgrade --install fluentd ./fluentd \
${OSH_INFRA_EXTRA_HELM_ARGS} \
${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD}
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh osh-infra