Gates: Update fluent-logging/elasticsearch configurations

This updates the fluentd buffer output configurations to account
for the restraints of the jobs deploying fluentd. This also
renames the fluentd configuration key from td_agent to fluentd to
reflect the fact we're no longer deploying td-agent

This also updates the Elasticsearch default replicas and overrides
the replica counts in each Elasticsearch deployment to account for
resource constraints

Change-Id: I55dee410eced99c3e1645f7452e4306ad646e601
This commit is contained in:
Steve Wilkerson 2018-10-19 07:57:35 -05:00
parent 538d51e991
commit 4c29bafcbc
7 changed files with 10 additions and 10 deletions

View File

@ -106,7 +106,7 @@ pod:
default: kubernetes.io/hostname default: kubernetes.io/hostname
replicas: replicas:
master: 3 master: 3
data: 2 data: 3
client: 3 client: 3
lifecycle: lifecycle:
upgrades: upgrades:

View File

@ -34,7 +34,7 @@ function check_output_indexes_exist () {
Elasticsearch outputs should match. This means we can safely use the last item Elasticsearch outputs should match. This means we can safely use the last item
in fluentd's configuration to get the Fluentd flush output interval in fluentd's configuration to get the Fluentd flush output interval
*/}} */}}
{{- $fluentdConf := last .Values.conf.td_agent -}} {{- $fluentdConf := last .Values.conf.fluentd -}}
{{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}} {{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}}
{{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}} {{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}}
fluentdFlush={{$fluentdFlush}} fluentdFlush={{$fluentdFlush}}

View File

@ -25,7 +25,7 @@ type: Opaque
data: data:
fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }} fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }}
parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }} parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }}
fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.td_agent | b64enc }} fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }}
{{ range $template, $fields := .Values.conf.templates }} {{ range $template, $fields := .Values.conf.templates }}
{{ $template }}.json: {{ toJson $fields | b64enc }} {{ $template }}.json: {{ toJson $fields | b64enc }}
{{ end }} {{ end }}

View File

@ -202,7 +202,7 @@ conf:
Time_Key: time Time_Key: time
Time_Format: "%Y-%m-%dT%H:%M:%S.%L" Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
Time_Keep: On Time_Keep: On
td_agent: fluentd:
- metrics_agent: - metrics_agent:
header: source header: source
type: monitor_agent type: monitor_agent
@ -304,10 +304,8 @@ conf:
buffer_chunk_limit: 2M buffer_chunk_limit: 2M
buffer_queue_limit: 8 buffer_queue_limit: 8
flush_interval: "5" flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300 max_retry_wait: 300
disable_retry_limit: "" disable_retry_limit: ""
num_threads: 8
- kernel_elasticsearch: - kernel_elasticsearch:
header: match header: match
type: elasticsearch type: elasticsearch
@ -322,10 +320,8 @@ conf:
buffer_chunk_limit: 2M buffer_chunk_limit: 2M
buffer_queue_limit: 8 buffer_queue_limit: 8
flush_interval: "5" flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300 max_retry_wait: 300
disable_retry_limit: "" disable_retry_limit: ""
num_threads: 8
# NOTE(srwilkers): This configuration entry should always be the last output # NOTE(srwilkers): This configuration entry should always be the last output
# defined, as it is used to determine the total flush cycle time for fluentbit # defined, as it is used to determine the total flush cycle time for fluentbit
# and fluentd # and fluentd
@ -342,10 +338,8 @@ conf:
buffer_chunk_limit: 2M buffer_chunk_limit: 2M
buffer_queue_limit: 8 buffer_queue_limit: 8
flush_interval: "5" flush_interval: "5"
flush_thread_count: 2
max_retry_wait: 300 max_retry_wait: 300
disable_retry_limit: "" disable_retry_limit: ""
num_threads: 8
type_name: fluent type_name: fluent
fluentd_exporter: fluentd_exporter:
log: log:

View File

@ -21,6 +21,10 @@ make elasticsearch
#NOTE: Deploy command #NOTE: Deploy command
tee /tmp/elasticsearch.yaml << EOF tee /tmp/elasticsearch.yaml << EOF
pod:
replicas:
data: 1
master: 2
conf: conf:
elasticsearch: elasticsearch:
env: env:

View File

@ -27,6 +27,7 @@ monitoring:
pod: pod:
replicas: replicas:
data: 1 data: 1
master: 2
manifests: manifests:
network_policy: true network_policy: true
network_policy: network_policy:

View File

@ -27,6 +27,7 @@ monitoring:
pod: pod:
replicas: replicas:
data: 1 data: 1
master: 2
conf: conf:
elasticsearch: elasticsearch:
snapshots: snapshots: