Gates: Update fluent-logging/elasticsearch configurations

This updates the fluentd buffer output configurations to account
for the restraints of the jobs deploying fluentd. This also
renames the fluentd configuration key from td_agent to fluentd to
reflect the fact we're no longer deploying td-agent

This also updates the Elasticsearch default replicas and overrides
the replica counts in each Elasticsearch deployment to account for
resource constraints

Change-Id: I55dee410eced99c3e1645f7452e4306ad646e601

elasticsearch/values.yaml

@@ -106,7 +106,7 @@ pod:
           default: kubernetes.io/hostname
   replicas:
     master: 3
-    data: 2
+    data: 3
     client: 3
   lifecycle:
     upgrades:

fluent-logging/templates/bin/_helm-tests.sh.tpl

@@ -34,7 +34,7 @@ function check_output_indexes_exist () {
     Elasticsearch outputs should match. This means we can safely use the last item
     in fluentd's configuration to get the Fluentd flush output interval
   */}}
-  {{- $fluentdConf := last .Values.conf.td_agent -}}
+  {{- $fluentdConf := last .Values.conf.fluentd -}}
   {{- $fluentdElasticsearchConf := index $fluentdConf "elasticsearch" -}}
   {{- $fluentdFlush := index $fluentdElasticsearchConf "flush_interval" -}}
   fluentdFlush={{$fluentdFlush}}

fluent-logging/templates/configmap-etc.yaml

@@ -25,7 +25,7 @@ type: Opaque
 data:
   fluent-bit.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.fluentbit | b64enc }}
   parsers.conf: {{ include "fluent_logging.utils.to_fluentbit_conf" .Values.conf.parsers | b64enc }}
-  fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.td_agent | b64enc }}
+  fluent.conf: {{ include "fluent_logging.utils.to_fluentd_conf" .Values.conf.fluentd | b64enc }}
 {{ range $template, $fields := .Values.conf.templates }}
   {{ $template }}.json: {{ toJson $fields | b64enc }}
 {{ end }}

fluent-logging/values.yaml

@@ -202,7 +202,7 @@ conf:
         Time_Key: time
         Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
         Time_Keep: On
-  td_agent:
+  fluentd:
     - metrics_agent:
         header: source
         type: monitor_agent
@@ -304,10 +304,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
     - kernel_elasticsearch:
         header: match
         type: elasticsearch
@@ -322,10 +320,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
     # NOTE(srwilkers): This configuration entry should always be the last output
     # defined, as it is used to determine the total flush cycle time for fluentbit
     # and fluentd
@@ -342,10 +338,8 @@ conf:
         buffer_chunk_limit: 2M
         buffer_queue_limit: 8
         flush_interval: "5"
-        flush_thread_count: 2
         max_retry_wait: 300
         disable_retry_limit: ""
-        num_threads: 8
         type_name: fluent
   fluentd_exporter:
     log:

tools/deployment/multinode/120-elasticsearch.sh

@@ -21,6 +21,10 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+pod:
+  replicas:
+    data: 1
+    master: 2
 conf:
   elasticsearch:
     env:

tools/deployment/network-policy/120-elasticsearch.sh

@@ -27,6 +27,7 @@ monitoring:
 pod:
   replicas:
     data: 1
+    master: 2
 manifests:
   network_policy: true
 network_policy:

tools/deployment/osh-infra-logging/050-elasticsearch.sh

@@ -27,6 +27,7 @@ monitoring:
 pod:
   replicas:
     data: 1
+    master: 2
 conf:
   elasticsearch:
     snapshots: