diff --git a/fluent-logging/Chart.yaml b/fluent-logging/Chart.yaml
new file mode 100644
index 000000000..e87238067
--- /dev/null
+++ b/fluent-logging/Chart.yaml
@@ -0,0 +1,25 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Fluentd
+name: fluent-logging
+version: 0.1.0
+home: http://www.fluentbit.io/
+sources:
+ - https://github.com/fluent/fluentbit
+ - https://github.com/fluent/fluentd
+ - https://git.openstack.org/cgit/openstack/openstack-helm-infra
+maintainers:
+ - name: OpenStack-Helm Authors
diff --git a/fluent-logging/README.rst b/fluent-logging/README.rst
new file mode 100644
index 000000000..375a70414
--- /dev/null
+++ b/fluent-logging/README.rst
@@ -0,0 +1,30 @@
+Fluentd-logging
+===============
+
+OpenStack-Helm defines a centralized logging mechanism to provide insight into
+the the state of the OpenStack services and infrastructure components as
+well as underlying kubernetes platform. Among the requirements for a logging
+platform, where log data can come from and where log data need to be delivered
+are very variable. To support various logging scenarios, OpenStack-Helm should
+provide a flexible mechanism to meet with certain operation needs. This chart
+proposes fast and lightweight log forwarder and full featured log aggregator
+complementing each other providing a flexible and reliable solution. Especially,
+Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log
+aggregator and processor.
+
+
+Mechanism
+---------
+
+Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering,
+aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on
+each node and mounts the /var/lib/docker/containers directory. The Docker
+container runtime engine directs events posted to stdout and stderr to this
+directory on the host. Fluent-bit then forward the contents of that directory to
+Fluentd. Fluentd runs as deployment at the designated nodes and expose service
+for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to
+the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs.
+Fluentd will then forward the results to Elasticsearch and to optionally kafka.
+Elasticsearch indexes the logs in a logstash-* index by default. kafka stores
+the logs in a 'logs' topic by default. Any external tool can then consume the
+'logs' topic.
diff --git a/fluent-logging/requirements.yaml b/fluent-logging/requirements.yaml
new file mode 100644
index 000000000..00b2a9554
--- /dev/null
+++ b/fluent-logging/requirements.yaml
@@ -0,0 +1,19 @@
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+ - name: helm-toolkit
+ repository: http://localhost:8879/charts/
+ version: 0.1.0
diff --git a/fluent-logging/templates/bin/_fluentd.sh.tpl b/fluent-logging/templates/bin/_fluentd.sh.tpl
new file mode 100644
index 000000000..0450572c1
--- /dev/null
+++ b/fluent-logging/templates/bin/_fluentd.sh.tpl
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+ exec /usr/sbin/td-agent
+}
+
+function stop () {
+ kill -TERM 1
+}
+
+$COMMAND
diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl
new file mode 100644
index 000000000..304dee0de
--- /dev/null
+++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+# Tests whether fluentd has successfully indexed data into Elasticsearch under
+# the logstash-* index via the fluent-elasticsearch plugin
+function check_logstash_index () {
+ total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \
+ | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
+ if [ "$total_hits" -gt 0 ]; then
+ echo "PASS: Successful hits on logstash-* index, provided by fluentd!"
+ else
+ echo "FAIL: No hits on query for logstash-* index! Exiting";
+ exit 1;
+ fi
+}
+
+# Tests whether fluentd has successfully tagged data with the kube.*
+# prefix via the fluent-kubernetes plugin
+function check_kubernetes_tag () {
+ total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?q=tag:kube.*" -H 'Content-Type: application/json' \
+ | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
+ if [ "$total_hits" -gt 0 ]; then
+ echo "PASS: Successful hits on logstash-* index, provided by fluentd!"
+ else
+ echo "FAIL: No hits on query for logstash-* index! Exiting";
+ exit 1;
+ fi
+}
+
+# Sleep for at least the buffer flush time to allow for indices to be populated
+sleep 30
+check_logstash_index
+check_kubernetes_tag
diff --git a/fluent-logging/templates/clusterrole.yaml b/fluent-logging/templates/clusterrole.yaml
new file mode 100644
index 000000000..7fe755db9
--- /dev/null
+++ b/fluent-logging/templates/clusterrole.yaml
@@ -0,0 +1,54 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.clusterrole }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: fluent-logging-runner
+rules:
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - namespaces
+ - nodes
+ - pods
+ - services
+ - replicationcontrollers
+ - limitranges
+ verbs:
+ - list
+ - watch
+ - apiGroups:
+ - extensions
+ resources:
+ - daemonsets
+ - deployments
+ - replicasets
+ verbs:
+ - list
+ - watch
+ - apiGroups:
+ - apps
+ resources:
+ - statefulsets
+ verbs:
+ - get
+ - list
+ - watch
+{{- end }}
diff --git a/fluent-logging/templates/clusterrolebinding.yaml b/fluent-logging/templates/clusterrolebinding.yaml
new file mode 100644
index 000000000..4d8f32005
--- /dev/null
+++ b/fluent-logging/templates/clusterrolebinding.yaml
@@ -0,0 +1,30 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.clusterrolebinding }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: run-fluent-logging
+subjects:
+ - kind: ServiceAccount
+ name: fluent-logging
+ namespace: {{ .Release.Namespace }}
+roleRef:
+ kind: ClusterRole
+ name: fluent-logging-runner
+ apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/fluent-logging/templates/configmap-bin.yaml b/fluent-logging/templates/configmap-bin.yaml
new file mode 100644
index 000000000..d95622d77
--- /dev/null
+++ b/fluent-logging/templates/configmap-bin.yaml
@@ -0,0 +1,31 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: fluent-logging-bin
+data:
+ fluentd.sh: |
+{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+ helm-tests.sh: |
+{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+ image-repo-sync.sh: |+
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}
diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml
new file mode 100644
index 000000000..75f46b8a6
--- /dev/null
+++ b/fluent-logging/templates/configmap-etc.yaml
@@ -0,0 +1,31 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_etc }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: fluent-logging-etc
+data:
+ fluent-bit.conf: |+
+{{- tuple .Values.conf.fluentbit "etc/_fluent-bit.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+ parsers.conf: |+
+{{- tuple .Values.conf.parsers "etc/_parsers.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+ td-agent.conf: |+
+{{- tuple .Values.conf.td_agent "etc/_td-agent.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+{{- end }}
diff --git a/fluent-logging/templates/daemonset-fluent-bit.yaml b/fluent-logging/templates/daemonset-fluent-bit.yaml
new file mode 100644
index 000000000..640b58f46
--- /dev/null
+++ b/fluent-logging/templates/daemonset-fluent-bit.yaml
@@ -0,0 +1,82 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.daemonset_fluentbit }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.fluentbit }}
+{{- if .Values.images.local_registry.active -}}
+{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentbit .Values.conditional_dependencies.local_image_registry) -}}
+{{- else -}}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentbit -}}
+{{- end -}}
+{{- $mounts_fluentbit := .Values.pod.mounts.fluentbit.fluentbit }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: fluentbit
+spec:
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+ template:
+ metadata:
+ labels:
+{{ tuple $envAll "fluentbit" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+ annotations:
+ configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+ spec:
+ serviceAccount: fluent-logging
+ nodeSelector:
+ {{ .Values.labels.fluentbit.node_selector_key }}: {{ .Values.labels.fluentbit.node_selector_value }}
+ hostNetwork: true
+ hostPID: true
+ dnsPolicy: ClusterFirstWithHostNet
+ initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+ containers:
+ - name: fluentbit
+ env:
+ image: {{ .Values.images.tags.fluentbit }}
+ imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+ volumeMounts:
+ - name: varlog
+ mountPath: /var/log
+ - name: varlibdockercontainers
+ mountPath: /var/lib/docker/containers
+ readOnly: true
+ - name: fluent-logging-etc
+ mountPath: /fluent-bit/etc/fluent-bit.conf
+ subPath: fluent-bit.conf
+ readOnly: true
+ - name: fluent-logging-etc
+ mountPath: /fluent-bit/etc/parsers.conf
+ subPath: parsers.conf
+ readOnly: true
+{{ if $mounts_fluentbit.volumeMounts }}{{ toYaml $mounts_fluentbit.volumeMounts | indent 8 }}{{ end }}
+ volumes:
+{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }}
+ - name: varlog
+ hostPath:
+ path: /var/log
+ - name: varlibdockercontainers
+ hostPath:
+ path: /var/lib/docker/containers
+ - name: fluent-logging-etc
+ configMap:
+ name: fluent-logging-etc
+ defaultMode: 0444
+{{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }}
+{{- end }}
diff --git a/fluent-logging/templates/deployment-fluentd.yaml b/fluent-logging/templates/deployment-fluentd.yaml
new file mode 100644
index 000000000..4bc84ac8a
--- /dev/null
+++ b/fluent-logging/templates/deployment-fluentd.yaml
@@ -0,0 +1,84 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.deployment_fluentd }}
+{{- $envAll := . }}
+{{- if .Values.images.local_registry.active -}}
+{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentd .Values.conditional_dependencies.local_image_registry) -}}
+{{- else -}}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentd -}}
+{{- end -}}
+{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: fluentd
+spec:
+ replicas: {{ .Values.pod.replicas.fluentd }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+ template:
+ metadata:
+ labels:
+{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+ annotations:
+ configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+ configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+ spec:
+ serviceAccount: fluent-logging
+ affinity:
+{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+ nodeSelector:
+ {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value }}
+ terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd_aggregator.timeout | default "30" }}
+ initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+ containers:
+ - name: fluentd
+ image: {{ .Values.images.tags.fluentd }}
+ imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+ command:
+ - /tmp/fluentd.sh
+ - start
+ ports:
+ - containerPort: {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+ volumeMounts:
+ - name: pod-etc-fluentd
+ mountPath: /etc/td-agent
+ - name: fluent-logging-etc
+ mountPath: /etc/td-agent/td-agent.conf
+ subPath: td-agent.conf
+ readOnly: true
+ - name: fluent-logging-bin
+ mountPath: /tmp/fluentd.sh
+ subPath: fluentd.sh
+ readOnly: true
+{{- if $mounts_fluentd.volumeMounts }}{{ toYaml $mounts_fluentd.volumeMounts | indent 12 }}{{- end }}
+ volumes:
+{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }}
+ - name: pod-etc-fluentd
+ emptyDir: {}
+ - name: fluent-logging-etc
+ configMap:
+ name: fluent-logging-etc
+ defaultMode: 0444
+ - name: fluent-logging-bin
+ configMap:
+ name: fluent-logging-bin
+ defaultMode: 0555
+{{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
+{{- end }}
diff --git a/fluent-logging/templates/etc/_fluent-bit.conf.tpl b/fluent-logging/templates/etc/_fluent-bit.conf.tpl
new file mode 100644
index 000000000..7b09615d9
--- /dev/null
+++ b/fluent-logging/templates/etc/_fluent-bit.conf.tpl
@@ -0,0 +1,19 @@
+[SERVICE]
+ Flush 1
+ Daemon Off
+ Log_Level {{ .Values.conf.fluentbit.service.log_level }}
+ Parsers_File parsers.conf
+
+[INPUT]
+ Name tail
+ Tag kube.*
+ Path /var/log/containers/*.log
+ Parser docker
+ DB /var/log/flb_kube.db
+ Mem_Buf_Limit {{ .Values.conf.fluentbit.input.mem_buf_limit }}
+
+[OUTPUT]
+ Name forward
+ Match *
+ Host {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+ Port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
diff --git a/fluent-logging/templates/etc/_parsers.conf.tpl b/fluent-logging/templates/etc/_parsers.conf.tpl
new file mode 100644
index 000000000..9f3b6b331
--- /dev/null
+++ b/fluent-logging/templates/etc/_parsers.conf.tpl
@@ -0,0 +1,6 @@
+[PARSER]
+ Name docker
+ Format json
+ Time_Key time
+ Time_Format %Y-%m-%dT%H:%M:%S.%L
+ Time_Keep On
diff --git a/fluent-logging/templates/etc/_td-agent.conf.tpl b/fluent-logging/templates/etc/_td-agent.conf.tpl
new file mode 100644
index 000000000..b9d78bbb2
--- /dev/null
+++ b/fluent-logging/templates/etc/_td-agent.conf.tpl
@@ -0,0 +1,83 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+<source>
+ @type forward
+ port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+ bind 0.0.0.0
+</source>
+
+<filter kube.**>
+ type kubernetes_metadata
+</filter>
+
+<match ** >
+{{ if .Values.conf.fluentd.kafka.enabled }}
+ @type copy
+
+ <store>
+ @type kafka_buffered
+
+ # list of seed brokers
+ brokers {{ tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "kafka" "public" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+
+ # buffer settings
+ buffer_type file
+ buffer_path /var/log/td-agent/buffer/td
+ flush_interval {{ .Values.conf.fluentd.kafka.flush_interval }}
+
+ # topic settings
+ default_topic {{ .Values.conf.fluentd.kafka.topic_name }}
+
+ # data type settings
+ output_data_type {{ .Values.conf.fluentd.kafka.output_data_type }}
+ compression_codec gzip
+
+ # producer settings
+ max_send_retries 1
+ required_acks -1
+ </store>
+
+ <store>
+{{- end }}
+ @type elasticsearch
+ include_tag_key true
+ host {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+ port {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+ logstash_format {{ .Values.conf.fluentd.elasticsearch.logstash }}
+
+ # Set the chunk limit the same as for fluentd-gcp.
+ buffer_chunk_limit {{ .Values.conf.fluentd.elasticsearch.buffer_chunk_limit }}
+
+ # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
+ buffer_queue_limit {{ .Values.conf.fluentd.elasticsearch.buffer_queue_limit }}
+
+ # Flush buffer every 30s to write to Elasticsearch
+ flush_interval {{ .Values.conf.fluentd.elasticsearch.flush_interval }}
+
+ # Never wait longer than 5 minutes between retries.
+ max_retry_wait {{ .Values.conf.fluentd.elasticsearch.max_retry_wait }}
+
+{{- if .Values.conf.fluentd.elasticsearch.disable_retry_limit }}
+
+ # Disable the limit on the number of retries (retry forever).
+ disable_retry_limit
+{{- end }}
+
+ # Use multiple threads for processing.
+ num_threads {{ .Values.conf.fluentd.elasticsearch.num_threads }}
+{{ if .Values.conf.fluentd.kafka.enabled }}
+ </store>
+{{- end }}
+
+</match>
diff --git a/fluent-logging/templates/job-image-repo-sync.yaml b/fluent-logging/templates/job-image-repo-sync.yaml
new file mode 100644
index 000000000..741d93616
--- /dev/null
+++ b/fluent-logging/templates/job-image-repo-sync.yaml
@@ -0,0 +1,65 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_image_repo_sync }}
+{{- $envAll := . }}
+{{- if .Values.images.local_registry.active -}}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.image_repo_sync -}}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: fluent-logging-image-repo-sync
+spec:
+ template:
+ metadata:
+ labels:
+{{ tuple $envAll "fluent-logging-exporter" "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+ spec:
+ restartPolicy: OnFailure
+ nodeSelector:
+ {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+ initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+ containers:
+ - name: image-repo-sync
+{{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.image_repo_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+ env:
+ - name: LOCAL_REPO
+ value: "{{ tuple "local_image_registry" "node" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "local_image_registry" "node" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
+ - name: IMAGE_SYNC_LIST
+ value: "{{ include "helm-toolkit.utils.image_sync_list" . }}"
+ command:
+ - /tmp/image-repo-sync.sh
+ volumeMounts:
+ - name: fluent-logging-exporter-bin
+ mountPath: /tmp/image-repo-sync.sh
+ subPath: image-repo-sync.sh
+ readOnly: true
+ - name: docker-socket
+ mountPath: /var/run/docker.sock
+ volumes:
+{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }}
+ - name: fluent-logging-bin
+ configMap:
+ name: fluent-logging-bin
+ defaultMode: 0555
+ - name: docker-socket
+ hostPath:
+ path: /var/run/docker.sock
+{{- end }}
+{{- end }}
diff --git a/fluent-logging/templates/pod-helm-tests.yaml b/fluent-logging/templates/pod-helm-tests.yaml
new file mode 100644
index 000000000..98349f052
--- /dev/null
+++ b/fluent-logging/templates/pod-helm-tests.yaml
@@ -0,0 +1,46 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.helm_tests }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: "{{.Release.Name}}-test"
+ annotations:
+ "helm.sh/hook": test-success
+spec:
+ restartPolicy: Never
+ containers:
+ - name: {{.Release.Name}}-helm-tests
+{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }}
+ command:
+ - /tmp/helm-tests.sh
+ env:
+ - name: ELASTICSEARCH_ENDPOINT
+ value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+ volumeMounts:
+ - name: fluent-logging-bin
+ mountPath: /tmp/helm-tests.sh
+ subPath: helm-tests.sh
+ readOnly: true
+ volumes:
+ - name: fluent-logging-bin
+ configMap:
+ name: fluent-logging-bin
+ defaultMode: 0555
+{{- end }}
diff --git a/fluent-logging/templates/rbac-entrypoint.yaml b/fluent-logging/templates/rbac-entrypoint.yaml
new file mode 100644
index 000000000..311712ea9
--- /dev/null
+++ b/fluent-logging/templates/rbac-entrypoint.yaml
@@ -0,0 +1,19 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.rbac_entrypoint }}
+{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_rbac"}}
+{{- end }}
diff --git a/fluent-logging/templates/service-fluentd.yaml b/fluent-logging/templates/service-fluentd.yaml
new file mode 100644
index 000000000..4a3aa63bb
--- /dev/null
+++ b/fluent-logging/templates/service-fluentd.yaml
@@ -0,0 +1,37 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service_fluentd }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+ ports:
+ - name: aggregator
+ port: {{ .Values.network.fluentd.port }}
+ {{ if .Values.network.fluentd.node_port.enabled }}
+ nodePort: {{ .Values.network.fluentd.node_port.port }}
+ {{ end }}
+ selector:
+{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+ {{ if .Values.network.fluentd.node_port.enabled }}
+ type: NodePort
+ {{ end }}
+{{- end }}
+
diff --git a/fluent-logging/templates/serviceaccount.yaml b/fluent-logging/templates/serviceaccount.yaml
new file mode 100644
index 000000000..8d09a19c1
--- /dev/null
+++ b/fluent-logging/templates/serviceaccount.yaml
@@ -0,0 +1,22 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.serviceaccount }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: fluent-logging
+{{- end }}
diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml
new file mode 100644
index 000000000..995c011fd
--- /dev/null
+++ b/fluent-logging/values.yaml
@@ -0,0 +1,230 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for fluentbit.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+release_group: null
+
+labels:
+ fluentd:
+ node_selector_key: openstack-control-plane
+ node_selector_value: enabled
+ fluentbit:
+ node_selector_key: openstack-control-plane
+ node_selector_value: enabled
+
+images:
+ tags:
+ fluentbit: docker.io/fluent/fluent-bit:0.12.9
+ fluentd: docker.io/kolla/ubuntu-source-fluentd:ocata
+ dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
+ helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3
+ image_repo_sync: docker.io/docker:17.07.0
+ pull_policy: IfNotPresent
+ local_registry:
+ active: false
+ exclude:
+ - dep_check
+ - image_repo_sync
+
+dependencies:
+ image_repo_sync:
+ services:
+ - service: local_image_registry
+ endpoint: internal
+ fluentd:
+ services:
+ - service: elasticsearch
+ endpoint: internal
+ fluentd_with_kafka:
+ services:
+ - service: elasticsearch
+ endpoint: internal
+ - service: kafka
+ endpoint: public
+ fluentbit:
+ services:
+ - service: aggregator
+ endpoint: internal
+ tests:
+ services:
+ - service: elasticsearch
+ endpoint: internal
+ - service: aggregator
+ endpoint: internal
+
+conditional_dependencies:
+ local_image_registry:
+ jobs:
+ - fluent-logging-image-repo-sync
+ services:
+ - service: local_image_registry
+ endpoint: node
+ fluentd:
+ services:
+ - service: kafka
+ endpoint: public
+
+
+conf:
+ fluentbit:
+ service:
+ log_level: info
+ input:
+ mem_buf_limit: 5MB
+ override:
+ fluentd:
+ kafka:
+ enabled: false
+ topic_name: logs
+ flush_interval: 3s
+ output_data_type: json
+ elasticsearch:
+ logstash: true
+ buffer_chunk_limit: 10M
+ buffer_queue_limit: 32
+ flush_interval: 15s
+ max_retry_wait: 300
+ disable_retry_limit: true
+ num_threads: 8
+ override:
+
+endpoints:
+ cluster_domain_suffix: cluster.local
+ elasticsearch:
+ namespace: null
+ name: elasticsearch
+ hosts:
+ data: elasticsearch-data
+ default: elasticsearch-logging
+ discovery: elasticsearch-discovery
+ public: elasticsearch
+ host_fqdn_override:
+ default: null
+ path:
+ default: null
+ scheme:
+ default: http
+ port:
+ client:
+ default: 9200
+ discovery:
+ default: 9300
+ kafka:
+ namespace: null
+ name: kafka
+ hosts:
+ default: kafka-logging
+ public: kafka
+ scheme:
+ default: http
+ public: http
+ port:
+ service:
+ default: 9092
+ aggregator:
+ namespace: null
+ name: fluentd
+ hosts:
+ default: fluentd-logging
+ internal: fluentd-logging
+ scheme:
+ default: http
+ port:
+ service:
+ default: 24224
+ internal: 24224
+ host_fqdn_override:
+ default: null
+
+network:
+ fluentd:
+ node_port:
+ enabled: false
+ port: 32329
+ port: 24224
+
+pod:
+ affinity:
+ anti:
+ type:
+ default: preferredDuringSchedulingIgnoredDuringExecution
+ topologyKey:
+ default: kubernetes.io/hostname
+ lifecycle:
+ upgrades:
+ daemonsets:
+ pod_replacement_strategy: RollingUpdate
+ fluentbit:
+ enabled: true
+ min_ready_seconds: 0
+ max_unavailable: 1
+ deployments:
+ revision_history: 3
+ pod_replacement_strategy: RollingUpdate
+ rolling_update:
+ max_unavailable: 1
+ max_surge: 3
+ termination_grace_period:
+ fluentd_aggregator:
+ timeout: 30
+ resources:
+ fluentbit:
+ enabled: false
+ limits:
+ memory: '400Mi'
+ cpu: '400m'
+ requests:
+ memory: '100Mi'
+ cpu: '100m'
+ fluentd:
+ enabled: false
+ limits:
+ memory: '1024Mi'
+ cpu: '2000m'
+ requests:
+ memory: '128Mi'
+ cpu: '500m'
+ jobs:
+ tests:
+ limits:
+ memory: '1024Mi'
+ cpu: '2000m'
+ requests:
+ memory: '128Mi'
+ cpu: '100m'
+ replicas:
+ fluentd: 3
+ mounts:
+ fluentd:
+ fluentd:
+ fluentbit:
+ fluentbit:
+ fluent_tests:
+ fluent_tests:
+
+manifests:
+ service_fluentd: true
+ deployment_fluentd: true
+ daemonset_fluentbit: true
+ job_image_repo_sync: true
+ helm_tests: true
+ configmap_bin: true
+ configmap_etc: true
+ clusterrole: true
+ clusterrolebinding: true
+ rbac_entrypoint: true
+ serviceaccount: true
diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml
index 3fb79c097..c356ae310 100644
--- a/tools/gate/chart-deploys/default.yaml
+++ b/tools/gate/chart-deploys/default.yaml
@@ -32,6 +32,7 @@ chart_groups:
timeout: 600
charts:
- openstack_elasticsearch
+ - fluent_logging
charts:
docker_registry_nfs_provisioner:
@@ -140,3 +141,13 @@ charts:
values:
storage:
enabled: false
+
+ fluent_logging:
+ chart_name: fluent-logging
+ release: fluent-logging
+ namespace: openstack
+ timeout: 300
+ test:
+ enabled: true
+ timeout: 300
+ output: false