From 45ac5fbe0f664dd572155c76fb51e9061e473ba7 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Wed, 18 Dec 2019 13:44:23 -0600 Subject: [PATCH] Update Elastic Beats versions to 7.1.0 This updates the Elastic Beats charts to 7.1.0 to keep them aligned with the Kibana and Elasticsearch chart versions, which is required for compatibility This also updates the experimental job to use the single node minikube deployment as opposed to the standard 5 node multinode deployment Change-Id: I4baba6ca2ea2f3785f11905138b67979a4501caa Signed-off-by: Steve Wilkerson --- elastic-filebeat/templates/configmap-etc.yaml | 4 +- elastic-filebeat/templates/daemonset.yaml | 13 ++ elastic-filebeat/values.yaml | 181 +++++++++++------- .../templates/daemonset-node-metrics.yaml | 10 +- .../templates/deployment-modules.yaml | 55 +++++- elastic-metricbeat/values.yaml | 88 +++++---- elastic-packetbeat/templates/daemonset.yaml | 5 +- elastic-packetbeat/values.yaml | 20 +- .../elastic-beats/005-deploy-k8s.sh | 1 + .../010-deploy-docker-registry.sh | 1 - .../elastic-beats/050-elasticsearch.sh | 29 ++- tools/deployment/elastic-beats/060-kibana.sh | 60 ++++++ ...etricbeat.sh => 080-elastic-metricbeat.sh} | 19 +- ...ic-filebeat.sh => 090-elastic-filebeat.sh} | 18 +- .../elastic-beats/100-elastic-apm-server.sh | 31 --- ...acketbeat.sh => 100-elastic-packetbeat.sh} | 19 +- tools/deployment/elastic-beats/110-kibana.sh | 1 - zuul.d/jobs.yaml | 19 +- zuul.d/project.yaml | 6 +- 19 files changed, 401 insertions(+), 179 deletions(-) create mode 120000 tools/deployment/elastic-beats/005-deploy-k8s.sh delete mode 120000 tools/deployment/elastic-beats/010-deploy-docker-registry.sh create mode 100755 tools/deployment/elastic-beats/060-kibana.sh rename tools/deployment/elastic-beats/{060-elastic-metricbeat.sh => 080-elastic-metricbeat.sh} (76%) rename tools/deployment/elastic-beats/{080-elastic-filebeat.sh => 090-elastic-filebeat.sh} (76%) delete mode 100755 tools/deployment/elastic-beats/100-elastic-apm-server.sh rename tools/deployment/elastic-beats/{090-elastic-packetbeat.sh => 100-elastic-packetbeat.sh} (76%) delete mode 120000 tools/deployment/elastic-beats/110-kibana.sh diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml index 2e2fc1232..29e448a3f 100644 --- a/elastic-filebeat/templates/configmap-etc.yaml +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -23,5 +23,7 @@ metadata: name: filebeat-etc data: filebeat.yml: | -{{ toYaml .Values.conf | indent 4 }} +{{ toYaml .Values.conf.filebeat | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} {{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml index 26f524136..e146c23bb 100644 --- a/elastic-filebeat/templates/daemonset.yaml +++ b/elastic-filebeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -105,10 +106,18 @@ spec: - name: filebeat containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: ELASTICSEARCH_HOST value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -133,6 +142,10 @@ spec: mountPath: /usr/share/filebeat/filebeat.yml readOnly: true subPath: filebeat.yml + - name: filebeat-etc + mountPath: /usr/share/filebeat/modules.d/system.yml + subPath: system.yml + readOnly: true {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} volumes: - name: pod-tmp diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml index 92289fd1d..c0c1b76f4 100644 --- a/elastic-filebeat/values.yaml +++ b/elastic-filebeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - filebeat: docker.elastic.co/beats/filebeat:6.2.3 + filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,82 +53,106 @@ dependencies: service: local_image_registry static: filebeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal service: local_image_registry conf: - setup: - dashboards: - enabled: true - path: - logs: /var/log/ - output: - elasticsearch: - hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] - username: "${ELASTICSEARCH_USERNAME}" - password: "${ELASTICSEARCH_PASSWORD}" filebeat: - config: - prospectors: - path: ${path.config}/prospectors.d/*.yml - reload: - enabled: false - modules: - path: ${path.config}/modules.d/*.yml - reload: - enabled: false - autodiscover: - providers: - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: kube-system - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: ceph - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: openstack - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - prospectors: - - type: docker - containers.ids: - - "*" - multiline: - pattern: '^Traceback' - match: after - negate: true - processors: - - add_kubernetes_metadata: - in_cluster: true - - drop_event: - when: - equals: - kubernetes: - container: - name: "filebeat" + setup: + dashboards: + enabled: true + index: "filebeat-*" + retry: + enabled: true + interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: osh-infra + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + modules: + system: + - module: system + syslog: + enabled: true + var.paths: ["/var/log/syslog*"] + fields: + host: + name: "${NODE_NAME}" + auth: + enabled: true + var.paths: ["/var/log/auth.log"] + fields: + host: + name: "${NODE_NAME}" endpoints: cluster_domain_suffix: cluster.local @@ -165,6 +189,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 filebeat: namespace: null name: filebeat diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml index 51595e2f3..9f47a8393 100644 --- a/elastic-metricbeat/templates/daemonset-node-metrics.yaml +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -48,8 +48,10 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges + - events verbs: - get - list @@ -104,7 +106,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" - "-system.hostfs=/hostfs" env: @@ -112,6 +114,10 @@ spec: value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -130,7 +136,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml index 4e1c60236..7ffadb674 100644 --- a/elastic-metricbeat/templates/deployment-modules.yaml +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -19,9 +19,54 @@ limitations under the License. {{- $esUserSecret := .Values.secrets.elasticsearch.user }} -{{- $serviceAccountName := "metricbeat" }} +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }} {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + - events + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -57,7 +102,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -68,6 +113,10 @@ spec: value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: KUBE_STATE_METRICS_PORT value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -86,7 +135,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml index c8a4a3e3c..636636268 100644 --- a/elastic-metricbeat/values.yaml +++ b/elastic-metricbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - metricbeat: docker.elastic.co/beats/metricbeat:6.3.1 + metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,7 +53,9 @@ dependencies: service: local_image_registry static: metricbeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal @@ -64,9 +66,14 @@ conf: setup: dashboards: enabled: true + index: metricbeat-* retry: enabled: true interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" metricbeat: config: modules: @@ -79,35 +86,20 @@ conf: username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} modules: - mysql: - - module: mysql - metricsets: ["status"] - hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"] - username: root - password: password - rabbitmq: - - module: rabbitmq - metricsets: ["node", "queue", "connection"] + docker: + - module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + - "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s enabled: true - period: 30s - hosts: ["rabbitmq.openstack.svc.cluster.local:15672"] - username: rabbitmq - password: password - # docker: - # - module: docker - # metricsets: - # - "container" - # - "cpu" - # - "diskio" - # - "healthcheck" - # - "info" - # - "image" - # - "memory" - # - "network" - # hosts: ["unix:///var/run/docker.sock"] - # period: 10s - # enabled: false - # labels.dedot: true system: - module: system period: 10s @@ -121,18 +113,15 @@ conf: - core - diskio - socket + - filesystem + - fsstat processes: ['.*'] + cpu.metrics: ["percentages"] + core.metrics: ["percentages"] process.include_top_n: by_cpu: 5 by_memory: 5 - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + enabled: true daemonset_kubernetes: - module: kubernetes metricsets: @@ -143,6 +132,9 @@ conf: - volume period: 10s hosts: ["localhost:10255"] + add_metadata: true + in_cluster: true + enabled: true deployment_kubernetes: - module: kubernetes metricsets: @@ -154,6 +146,9 @@ conf: - event period: 10s hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + add_metadata: true + in_cluster: true + enabled: true endpoints: cluster_domain_suffix: cluster.local @@ -203,6 +198,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: osh-infra + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 pod: affinity: diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml index 6db4cdac4..7978b652c 100644 --- a/elastic-packetbeat/templates/daemonset.yaml +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -102,7 +103,7 @@ spec: - NET_ADMIN args: - "-c" - - "/etc/packetbeat/packetbeat.yml" + - "/usr/share/packetbeat/packetbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -131,7 +132,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: packetbeat-etc - mountPath: /etc/packetbeat/packetbeat.yml + mountPath: /usr/share/packetbeat/packetbeat.yml subPath: packetbeat.yml readOnly: true {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml index 78955e429..2b163ed8f 100644 --- a/elastic-packetbeat/values.yaml +++ b/elastic-packetbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - packetbeat: docker.elastic.co/beats/packetbeat:6.3.1 + packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -63,28 +63,30 @@ conf: packetbeat: setup: kibana: - host: ['${KIBANA_HOST}:${KIBANA_PORT}'] + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" dashboards: enabled: true + index: "packetbeat-*" retry: enabled: true interval: 5 packetbeat: + flows: + timeout: 30s + period: 10s interfaces: device: any protocols: + - type: dhcpv4 + ports: [67, 68] - type: dns ports: [53] include_authorities: true include_additionals: true - type: http - ports: [80, 8080, 8081, 5000, 8002] - - type: memcache - ports: [11211] - - type: mysql - ports: [3306] - - type: rabbit - ports: [5672, 15672] + ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134] output: elasticsearch: hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] diff --git a/tools/deployment/elastic-beats/005-deploy-k8s.sh b/tools/deployment/elastic-beats/005-deploy-k8s.sh new file mode 120000 index 000000000..257a39f7a --- /dev/null +++ b/tools/deployment/elastic-beats/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh deleted file mode 120000 index 69d244e73..000000000 --- a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh index a0a227394..0313c64e7 100755 --- a/tools/deployment/elastic-beats/050-elasticsearch.sh +++ b/tools/deployment/elastic-beats/050-elasticsearch.sh @@ -21,12 +21,39 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +manifests: + cron_curator: false + configmap_bin_curator: false + configmap_etc_curator: false +images: + tags: + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115 +storage: + data: + requests: + storage: 20Gi + master: + requests: + storage: 5Gi +jobs: + verify_repositories: + cron: "*/10 * * * *" monitoring: prometheus: - enabled: true + enabled: false pod: replicas: + client: 1 data: 1 + master: 2 +conf: + elasticsearch: + config: + xpack: + security: + enabled: false + ilm: + enabled: false EOF helm upgrade --install elasticsearch ./elasticsearch \ diff --git a/tools/deployment/elastic-beats/060-kibana.sh b/tools/deployment/elastic-beats/060-kibana.sh new file mode 100755 index 000000000..ac955075b --- /dev/null +++ b/tools/deployment/elastic-beats/060-kibana.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make kibana + +: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} + +tee /tmp/kibana.yaml << EOF +images: + tags: + kibana: docker.elastic.co/kibana/kibana:7.1.0 +conf: + kibana: + xpack: + security: + enabled: false + spaces: + enabled: false + apm: + enabled: false + graph: + enabled: false + ml: + enabled: false + monitoring: + enabled: false + reporting: + enabled: false + canvas: + enabled: false +EOF + +#NOTE: Deploy command +helm upgrade --install kibana ./kibana \ + --namespace=osh-infra \ + --values=/tmp/kibana.yaml + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status kibana diff --git a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/060-elastic-metricbeat.sh rename to tools/deployment/elastic-beats/080-elastic-metricbeat.sh index 1fe6cda39..cf3c73400 100755 --- a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh +++ b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-metricbeat +tee /tmp/metricbeat.yaml << EOF +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat:7.1.0 +conf: + metricbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ --namespace=kube-system \ - --set endpoints.kube_state_metrics.namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/metricbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/080-elastic-filebeat.sh b/tools/deployment/elastic-beats/090-elastic-filebeat.sh similarity index 76% rename from tools/deployment/elastic-beats/080-elastic-filebeat.sh rename to tools/deployment/elastic-beats/090-elastic-filebeat.sh index cb36c71c2..f73848095 100755 --- a/tools/deployment/elastic-beats/080-elastic-filebeat.sh +++ b/tools/deployment/elastic-beats/090-elastic-filebeat.sh @@ -19,10 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-filebeat +tee /tmp/filebeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/filebeat:7.1.0 +conf: + filebeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-filebeat ./elastic-filebeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/filebeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/100-elastic-apm-server.sh b/tools/deployment/elastic-beats/100-elastic-apm-server.sh deleted file mode 100755 index 1e4f70513..000000000 --- a/tools/deployment/elastic-beats/100-elastic-apm-server.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-apm-server - -#NOTE: Deploy command -helm upgrade --install elastic-apm-server ./elastic-apm-server \ - --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system - -#NOTE: Validate Deployment info -helm status elastic-apm-server diff --git a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/090-elastic-packetbeat.sh rename to tools/deployment/elastic-beats/100-elastic-packetbeat.sh index 0b596ef40..c0fc08672 100755 --- a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh +++ b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-packetbeat +tee /tmp/packetbeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/packetbeat:7.1.0 +conf: + packetbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra \ - --set endpoints.kibana.namespace=osh-infra + --values=/tmp/packetbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/110-kibana.sh b/tools/deployment/elastic-beats/110-kibana.sh deleted file mode 120000 index 03852eb06..000000000 --- a/tools/deployment/elastic-beats/110-kibana.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/075-kibana.sh \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8385a59e8..1026c9e00 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -348,29 +348,24 @@ - job: name: openstack-helm-infra-elastic-beats parent: openstack-helm-infra-functional - nodeset: openstack-helm-five-node-ubuntu timeout: 7200 - pre-run: - - playbooks/osh-infra-upgrade-host.yaml - - playbooks/osh-infra-deploy-docker.yaml - - playbooks/osh-infra-build.yaml - - playbooks/osh-infra-deploy-k8s.yaml + pre-run: playbooks/osh-infra-upgrade-host.yaml run: playbooks/osh-infra-gate-runner.yaml post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node vars: gate_scripts: - - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh + - ./tools/deployment/elastic-beats/005-deploy-k8s.sh - ./tools/deployment/elastic-beats/020-ingress.sh - ./tools/deployment/elastic-beats/030-ceph.sh - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh - ./tools/deployment/elastic-beats/040-ldap.sh - ./tools/deployment/elastic-beats/050-elasticsearch.sh - - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/060-kibana.sh - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh - - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh - - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh - - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh - - ./tools/deployment/elastic-beats/110-kibana.sh + - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh + - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh - job: name: openstack-helm-infra-armada-deploy diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index ec3e29410..5b95b3ec6 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -57,15 +57,15 @@ # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords - # experimental: - # jobs: + experimental: + jobs: # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved #- openstack-helm-infra-five-fedora # NOTE(srwilkers): Disable centos experimental jobs until issues resolved #- openstack-helm-infra-five-centos # - openstack-helm-infra-five-ubuntu + - openstack-helm-infra-elastic-beats # - openstack-helm-infra-tenant-ceph - # - openstack-helm-infra-elastic-beats # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords