Elasticsearch, Fluent-logging, Kibana Ingress Policy

This adds ingress network policy for the fluent-logging, kibana
and Elasticsearch charts. This leverages the helm-toolkit template
that was used in openstack-helm for the openstack services

Change-Id: I2a89b62f1002851346e9a25de40113078e9c518f

elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml

@@ -27,17 +27,17 @@ kind: Deployment
 metadata:
   name: prometheus-elasticsearch-exporter
   labels:
-{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
   replicas: {{ .Values.pod.replicas.prometheus_elasticsearch_exporter }}
   selector:
     matchLabels:
-{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
 {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
   template:
     metadata:
       labels:
-{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
     spec:

elasticsearch/templates/monitoring/prometheus/exporter-network-policy.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-elasticsearch-exporter" -}}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- end -}}

elasticsearch/templates/monitoring/prometheus/exporter-service.yaml

@@ -23,7 +23,7 @@ kind: Service
 metadata:
   name: {{ tuple "prometheus_elasticsearch_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
   labels:
-{{ tuple $envAll "elasticsearch-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-elasticsearch-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
 {{- if .Values.monitoring.prometheus.enabled }}
 {{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
@@ -33,5 +33,5 @@ spec:
   - name: metrics
     port: {{ tuple "prometheus_elasticsearch_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
-{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}

elasticsearch/values.yaml

@@ -249,6 +249,14 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
+network_policy:
+  elasticsearch:
+    ingress:
+      - {}
+  prometheus-elasticsearch-exporter:
+    ingress:
+      - {}
+
 secrets:
   rgw:
     admin: radosgw-s3-admin-creds
@@ -703,6 +711,7 @@ manifests:
     prometheus:
       configmap_bin_exporter: true
       deployment_exporter: true
+      network_policy_exporter: false
       service_exporter: true
   network_policy: false
   service_data: true

fluent-logging/templates/job-elasticsearch-template.yaml

@@ -33,7 +33,7 @@ spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "fluent" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure

fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }}
+{{- if and .Values.manifests.monitoring.prometheus.configmap_bin_exporter .Values.monitoring.prometheus.enabled }}
 {{- $envAll := . }}
 ---
 apiVersion: v1

fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml

@@ -29,17 +29,17 @@ kind: Deployment
 metadata:
   name: prometheus-fluentd-exporter
   labels:
-{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
   replicas: {{ .Values.pod.replicas.prometheus_fluentd_exporter }}
   selector:
     matchLabels:
-{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
 {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
   template:
     metadata:
       labels:
-{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "fluentd_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}

fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2017-2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- end -}}

fluent-logging/templates/monitoring/prometheus/exporter-service.yaml

@@ -23,7 +23,7 @@ kind: Service
 metadata:
   name: {{ tuple "prometheus_fluentd_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
   labels:
-{{ tuple $envAll "prometheus_fluentd_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-fluentd-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
 {{- if .Values.monitoring.prometheus.enabled }}
 {{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
@@ -33,5 +33,5 @@ spec:
   - name: metrics
     port: {{ tuple "prometheus_fluentd_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   selector:
-{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "prometheus-fluentd-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}

fluent-logging/templates/network-policy.yaml

@@ -14,12 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License. */}}
 
 {{- if .Values.manifests.network_policy -}}
-{{- $netpol_opts := dict "envAll" . "name" "application" "label" "fluentbit" }}
-{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
 {{ $netpol_opts := dict "envAll" . "name" "application" "label" "fluentd" }}
 {{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
-{{ $netpol_opts := dict "envAll" . "name" "application" "label" "fluent" }}
-{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
-{{ $netpol_opts := dict "envAll" . "name" "application" "label" "fluent-logging" }}
-{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
 {{- end -}}

fluent-logging/templates/pod-helm-tests.yaml

@@ -26,7 +26,7 @@ kind: Pod
 metadata:
   name: "{{.Release.Name}}-test"
   labels:
-{{ tuple $envAll "fluent-logging" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
     "helm.sh/hook": test-success
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}

fluent-logging/values.yaml

@@ -570,6 +570,14 @@ network:
       enabled: false
       port: 32329
 
+network_policy:
+  prometheus-fluentd-exporter:
+    ingress:
+      - {}
+  fluentd:
+    ingress:
+      - {}
+
 pod:
   security_context:
     fluentd:
@@ -678,8 +686,9 @@ manifests:
   helm_tests: true
   monitoring:
     prometheus:
-      configmap_bin: true
+      configmap_bin_exporter: true
       deployment_exporter: true
+      network_policy_exporter: false
       service_exporter: true
   network_policy: false
   secret_elasticsearch: true

kibana/values.yaml

@@ -83,6 +83,11 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
+network_policy:
+  kibana:
+    ingress:
+      - {}
+
 secrets:
   elasticsearch:
     user: kibana-elasticsearch-user

tools/deployment/network-policy/120-elasticsearch.sh

@@ -21,9 +21,46 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
-monitoring:
+network_policy:
-  prometheus:
+  prometheus-elasticsearch-exporter:
-    enabled: true
+    ingress:
+      - from:
+        - podSelector:
+            matchLabels:
+              application: prometheus
+        ports:
+        - protocol: TCP
+          port: 9108
+  elasticsearch:
+    ingress:
+      - from:
+        - podSelector:
+            matchLabels:
+              application: elasticsearch
+        - podSelector:
+            matchLabels:
+              application: prometheus-elasticsearch-exporter
+        - podSelector:
+            matchLabels:
+              application: fluentd
+        - podSelector:
+            matchLabels:
+              application: ingress
+        - podSelector:
+            matchLabels:
+              application: kibana
+        - podSelector:
+            matchLabels:
+              application: nagios
+        ports:
+        - protocol: TCP
+          port: 80
+        - protocol: TCP
+          port: 443
+        - protocol: TCP
+          port: 9200
+        - protocol: TCP
+          port: 9300
 pod:
   replicas:
     data: 1
@@ -53,12 +90,14 @@ conf:
               timestring: '%Y.%m.%d'
               unit: days
               unit_count: 365
+monitoring:
+  prometheus:
+    enabled: true
 manifests:
   network_policy: true
-network_policy:
+  monitoring:
-  elasticsearch:
+    prometheus:
-    ingress:
+      network_policy_exporter: true
-      - from:
 EOF
 
 helm upgrade --install elasticsearch ./elasticsearch \

tools/deployment/network-policy/130-fluent-logging.sh

@@ -19,30 +19,153 @@ set -xe
 #NOTE: Lint and package chart
 make fluent-logging
 
-tee /tmp/fluent-logging.yaml <<EOF
+if [ ! -d "/var/log/journal" ]; then
+tee /tmp/fluent-logging.yaml << EOF
+pod:
+  replicas:
+    fluentd: 1
+monitoring:
+  prometheus:
+    enabled: true
 manifests:
   network_policy: true
-network_policy:
+  monitoring:
+    prometheus:
+      network_policy_exporter: true
+mounts:
   fluentbit:
+    fluentbit:
+      volumes:
+        - name: runlog
+          hostPath:
+            path: /run/log
+      volumeMounts:
+        - name: runlog
+          mountPath: /run/log
+network_policy:
+  prometheus-fluentd-exporter:
     ingress:
       - from:
+        - podSelector:
+            matchLabels:
+              application: prometheus
+        ports:
+        - protocol: TCP
+          port: 9309
   fluentd:
     ingress:
       - from:
-  fluent:
+        - podSelector:
-    ingress:
+            matchLabels:
-      - from:
+              application: fluentbit
-  fluent-logging:
+        - podSelector:
-    ingress:
+            matchLabels:
-      - from:
+              application: prometheus-fluentd-exporter
+        - podSelector:
+            matchLabels:
+              application: keystone
+        - podSelector:
+            matchLabels:
+              application: heat
+        - podSelector:
+            matchLabels:
+              application: glance
+        - podSelector:
+            matchLabels:
+              application: cinder
+        - podSelector:
+            matchLabels:
+              application: barbican
+        - podSelector:
+            matchLabels:
+              application: ironic
+        - podSelector:
+            matchLabels:
+              application: nova
+        - podSelector:
+            matchLabels:
+              application: neutron
+        - podSelector:
+            matchLabels:
+              application: placement
+        ports:
+        - protocol: TCP
+          port: 24224
+        - protocol: TCP
+          port: 24220
 EOF
-
-
-#NOTE: Deploy command
 helm upgrade --install fluent-logging ./fluent-logging \
     --namespace=osh-infra \
-    --values=/tmp/fluent-logging.yaml \
+    --values=/tmp/fluent-logging.yaml
-    --set pod.replicas.fluentd=1
+else
+tee /tmp/fluent-logging.yaml << EOF
+pod:
+  replicas:
+    fluentd: 1
+monitoring:
+  prometheus:
+    enabled: true
+manifests:
+  network_policy: true
+  monitoring:
+    prometheus:
+      network_policy_exporter: true
+network_policy:
+  prometheus-fluentd-exporter:
+    ingress:
+      - from:
+        - podSelector:
+            matchLabels:
+              application: prometheus
+        ports:
+        - protocol: TCP
+          port: 9309
+  fluentd:
+    ingress:
+      - from:
+        - podSelector:
+            matchLabels:
+              application: fluentbit
+        - podSelector:
+            matchLabels:
+              application: prometheus-fluentd-exporter
+        - podSelector:
+            matchLabels:
+              application: keystone
+        - podSelector:
+            matchLabels:
+              application: heat
+        - podSelector:
+            matchLabels:
+              application: glance
+        - podSelector:
+            matchLabels:
+              application: cinder
+        - podSelector:
+            matchLabels:
+              application: barbican
+        - podSelector:
+            matchLabels:
+              application: ironic
+        - podSelector:
+            matchLabels:
+              application: nova
+        - podSelector:
+            matchLabels:
+              application: neutron
+        - podSelector:
+            matchLabels:
+              application: placement
+        ports:
+        - protocol: TCP
+          port: 24224
+        - protocol: TCP
+          port: 24220
+EOF
+helm upgrade --install fluent-logging ./fluent-logging \
+    --namespace=osh-infra \
+    --values=/tmp/fluent-logging.yaml
+fi
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra

tools/deployment/network-policy/140-kibana.sh

@@ -19,27 +19,35 @@ set -xe
 #NOTE: Lint and package chart
 make kibana
 
-tee /tmp/kibana.yaml <<EOF
+#NOTE: Deploy command
-manifests:
+tee /tmp/kibana.yaml << EOF
-  network_policy: true
 network_policy:
   kibana:
     ingress:
       - from:
+        - podSelector:
+            matchLabels:
+              application: elasticsearch
         - podSelector:
             matchLabels:
               application: kibana
+        - podSelector:
+            matchLabels:
+              application: ingress
         ports:
-        - protocol: TCP
-          port: 5601
         - protocol: TCP
           port: 80
+        - protocol: TCP
+          port: 443
+        - protocol: TCP
+          port: 5601
+manifests:
+  network_policy: true
 EOF
-
-#NOTE: Deploy command
 helm upgrade --install kibana ./kibana \
     --namespace=osh-infra \
     --values=/tmp/kibana.yaml
+
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh osh-infra
 

tools/deployment/network-policy/901-test-networkpolicy.sh

@@ -55,6 +55,13 @@ test_netpol osh-infra mariadb server nagios.osh-infra.svc.cluster.local fail
 test_netpol osh-infra mariadb server prometheus.osh-infra.svc.cluster.local fail
 test_netpol osh-infra mariadb server nagios.osh-infra.svc.cluster.local fail
 test_netpol osh-infra mariadb server openstack-metrics.openstack.svc.cluster.local:9103 fail
+test_netpol osh-infra mariadb server kibana.osh-infra.svc.cluster.local fail
+test_netpol osh-infra mariadb server fluentd-logging.osh-infra.svc.cluster.local:24224 fail
+test_netpol osh-infra fluentbit daemon prometheus.osh-infra.svc.cluster.local fail
 
 # Doing positive tests
 test_netpol osh-infra grafana dashboard mariadb.osh-infra.svc.cluster.local:3306 success
+test_netpol osh-infra elasticsearch client kibana-dash.osh-infra.svc.cluster.local success
+test_netpol osh-infra fluentd internal elasticsearch-logging.osh-infra.svc.cluster.local success
+test_netpol osh-infra prometheus api fluentd-exporter.osh-infra.svc.cluster.local:9309/metrics success
+test_netpol osh-infra prometheus api elasticsearch-exporter.osh-infra.svc.cluster.local:9108/metrics success