Merge "Ceph: Remove fluentbit sidecars, mount hostpath for logs"

This commit is contained in:
Zuul 2018-10-17 21:41:38 +00:00 committed by Gerrit Code Review
commit cd4b8e9b87
15 changed files with 70 additions and 330 deletions

@ -1,19 +0,0 @@
#!/bin/sh
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

@ -60,8 +60,4 @@ data:
utils-checkPGs.sh: |
{{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ if .Values.logging.fluentd }}
fluentbit-sidecar.sh: |
{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ end }}
{{- end }}

@ -42,15 +42,6 @@ limitations under the License.
{{- if empty .Values.conf.ceph.osd.public_network -}}
{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
{{- end -}}
{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
{{- end -}}
---
apiVersion: v1
kind: ConfigMap
@ -59,12 +50,6 @@ metadata:
data:
ceph.conf: |
{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
{{ if .Values.logging.fluentd }}
fluent-bit.conf: |
{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
parsers.conf: |
{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
{{ end }}
{{- end }}
{{- end }}
{{- end }}

@ -91,6 +91,19 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: ceph-log-ownership
{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
securityContext:
runAsUser: 0
command:
- chown
- -R
- ceph:root
- /var/log/ceph
volumeMounts:
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
containers:
- name: ceph-mon
{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }}
@ -195,33 +208,13 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: varlog
- name: pod-var-log
mountPath: /var/log/ceph
{{ if .Values.logging.fluentd }}
- name: fluentbit-sidecar
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/fluentbit-sidecar.sh
volumeMounts:
- name: ceph-mon-bin
mountPath: /tmp/fluentbit-sidecar.sh
subPath: fluentbit-sidecar.sh
readOnly: true
- name: varlog
mountPath: /var/log/ceph
- name: ceph-mon-etc
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
readOnly: true
- name: ceph-mon-etc
mountPath: /fluent-bit/etc/parsers.conf
subPath: parsers.conf
readOnly: true
{{ end }}
readOnly: false
volumes:
- name: varlog
emptyDir: {}
- name: pod-var-log
hostPath:
path: /var/log/ceph
- name: ceph-mon-bin
configMap:
name: ceph-mon-bin

@ -1,38 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This function generates fluentbit configuration files with entries in the
# ceph-mon values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# [HEADER]
# key value
# key value
# key value
# The configuration schema can be found here:
# http://fluentbit.io/documentation/0.12/configuration/schema.html
{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
{{- range $values := . -}}
{{- range $section := . -}}
{{- $header := pick . "header" -}}
{{- $config := omit . "header" }}
[{{$header.header | upper }}]
{{range $key, $value := $config -}}
{{ $key | indent 4 }} {{ $value }}
{{end -}}
{{- end -}}
{{- end -}}
{{- end -}}

@ -28,7 +28,6 @@ images:
ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
fluentbit: docker.io/fluent/fluent-bit:0.12.14
dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
image_repo_sync: docker.io/docker:17.07.0
local_registry:
@ -71,13 +70,6 @@ pod:
limits:
memory: "50Mi"
cpu: "500m"
fluentbit:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
jobs:
bootstrap:
limits:
@ -187,40 +179,6 @@ conf:
storage:
mon:
directory: /var/lib/openstack-helm/ceph/mon
fluentbit:
- service:
header: service
Flush: 30
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- ceph_tail:
# NOTE(srwilkers): Despite being exposed, these values should not be
# modified, as the ceph-mon logs are always placed here
header: input
Name: tail
Tag: ceph-mon.*
Path: /var/log/ceph/*.log
Parser: syslog
DB: /var/log/ceph/ceph.db
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
Refresh_Interval: 10s
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
logging:
fluentd: false
dependencies:
dynamic:
@ -320,22 +278,6 @@ endpoints:
port:
mon:
default: 6789
fluentd:
namespace: null
name: fluentd
hosts:
default: fluentd-logging
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 24224
metrics:
default: 24220
manifests:
configmap_bin: true

@ -1,20 +0,0 @@
#!/bin/sh
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf

@ -40,8 +40,4 @@ data:
{{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
init-dirs.sh: |
{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ if .Values.logging.fluentd }}
fluentbit-sidecar.sh: |
{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ end }}
{{- end }}

@ -40,15 +40,6 @@ limitations under the License.
{{- if empty .Values.conf.ceph.osd.public_network -}}
{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
{{- end -}}
{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
{{- end -}}
---
apiVersion: v1
kind: ConfigMap
@ -57,12 +48,6 @@ metadata:
data:
ceph.conf: |
{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
{{ if .Values.logging.fluentd }}
fluent-bit.conf: |
{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
parsers.conf: |
{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
{{ end }}
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}

@ -75,6 +75,35 @@ spec:
- name: pod-run
mountPath: /run
readOnly: false
- name: ceph-log-ownership
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
securityContext:
runAsUser: 0
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render and added to all containers in the
# pod
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
- name: CEPH_GET_ADMIN_KEY
value: "1"
command:
- chown
- -R
- ceph:root
- /var/log/ceph
volumeMounts:
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
- name: osd-init
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -218,30 +247,9 @@ spec:
- name: journal
mountPath: /var/lib/ceph/journal
readOnly: false
- name: varlog
- name: pod-var-log
mountPath: /var/log/ceph
{{ if .Values.logging.fluentd }}
- name: fluentbit-sidecar
{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/fluentbit-sidecar.sh
volumeMounts:
- name: ceph-osd-bin
mountPath: /tmp/fluentbit-sidecar.sh
subPath: fluentbit-sidecar.sh
readOnly: true
- name: varlog
mountPath: /var/log/ceph
- name: ceph-osd-etc
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
readOnly: true
- name: ceph-osd-etc
mountPath: /fluent-bit/etc/parsers.conf
subPath: parsers.conf
readOnly: true
{{ end }}
readOnly: false
volumes:
- name: devices
hostPath:
@ -251,8 +259,9 @@ spec:
path: /run/lvm
- name: pod-var-lib-ceph
emptyDir: {}
- name: varlog
emptyDir: {}
- name: pod-var-log
hostPath:
path: /var/log/ceph
- name: pod-run
emptyDir:
medium: "Memory"

@ -1,38 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
# This function generates fluentbit configuration files with entries in the
# ceph-osd values.yaml. It results in a configuration section with the
# following format (for as many key/value pairs defined in values for a section):
# [HEADER]
# key value
# key value
# key value
# The configuration schema can be found here:
# http://fluentbit.io/documentation/0.12/configuration/schema.html
{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
{{- range $values := . -}}
{{- range $section := . -}}
{{- $header := pick . "header" -}}
{{- $config := omit . "header" }}
[{{$header.header | upper }}]
{{range $key, $value := $config -}}
{{ $key | indent 4 }} {{ $value }}
{{end -}}
{{- end -}}
{{- end -}}
{{- end -}}

@ -22,7 +22,6 @@ images:
tags:
ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
fluentbit: docker.io/fluent/fluent-bit:0.12.14
dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
image_repo_sync: docker.io/docker:17.07.0
local_registry:
@ -56,13 +55,6 @@ pod:
limits:
memory: "1024Mi"
cpu: "1000m"
fluentbit:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
jobs:
image_repo_sync:
requests:
@ -148,40 +140,6 @@ conf:
# journal:
# type: directory
# location: /var/lib/openstack-helm/ceph/osd/journal-three
fluentbit:
- service:
header: service
Flush: 30
Daemon: Off
Log_Level: info
Parsers_File: parsers.conf
- ceph_tail:
# NOTE(srwilkers): Despite being exposed, these values should not be
# modified, as the ceph-osd logs are always placed here
header: input
Name: tail
Tag: ceph-osd.*
Path: /var/log/ceph/ceph-osd.**.log
Parser: syslog
DB: /var/log/ceph/ceph-osd.db
DB.Sync: Normal
Buffer_Chunk_Size: 1M
Buffer_Max_Size: 1M
Mem_Buf_Limit: 5MB
Refresh_Interval: 10s
parsers:
- syslog:
header: parser
Name: syslog
Format: regex
Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On
Types: "pid:integer"
logging:
fluentd: false
dependencies:
dynamic:
@ -242,22 +200,6 @@ endpoints:
port:
mon:
default: 6789
fluentd:
namespace: null
name: fluentd
hosts:
default: fluentd-logging
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
service:
default: 24224
metrics:
default: 24220
manifests:
configmap_bin: true

@ -106,8 +106,8 @@ pod:
default: kubernetes.io/hostname
replicas:
master: 3
data: 3
client: 2
data: 2
client: 3
lifecycle:
upgrades:
deployments:

@ -62,6 +62,8 @@ section):
{{ $type | indent 4 }} {{ $innerValue }}
{{- else if contains "ENV" ($innerValue | quote) }}
{{ $innerKey | indent 4 }} {{ $innerValue | quote }}
{{- else if eq $innerKey "flush_interval" }}
{{ $innerKey | indent 4 }} {{ printf "%ss" $innerValue }}
{{- else }}
{{ $innerKey | indent 4 }} {{ $innerValue }}
{{- end }}
@ -75,6 +77,8 @@ section):
{{ $type | indent 2 }} {{ $value }}
{{- else if contains "ENV" ($value | quote) }}
{{ $key | indent 2 }} {{ $value | quote }}
{{- else if eq $key "flush_interval" }}
{{ $key | indent 2 }} {{ printf "%ss" $value }}
{{- else }}
{{ $key | indent 2 }} {{ $value }}
{{- end -}}

@ -301,9 +301,10 @@ conf:
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: journal
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
@ -318,9 +319,10 @@ conf:
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
logstash_prefix: kernel
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8
@ -337,9 +339,10 @@ conf:
host: "#{ENV['ELASTICSEARCH_HOST']}"
port: "#{ENV['ELASTICSEARCH_PORT']}"
logstash_format: true
buffer_chunk_limit: 10M
buffer_queue_limit: 32
flush_interval: 20
buffer_chunk_limit: 2M
buffer_queue_limit: 8
flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300
disable_retry_limit: ""
num_threads: 8