Merge "Infrastructure and Cluster Monitoring"

stx-monitor-helm/centos/build_srpm.data

@@ -0,0 +1,6 @@
+SRC_DIR="stx-monitor-helm"
+# This version is used as a component of the stx-monitor application
+# version. Any change to this version must also be reflected in the
+# SUPPORTED_VERSIONS list in sysinv/helm/monitor_version_check.py
+#
+TIS_PATCH_VER=1

stx-monitor-helm/centos/stx-monitor-helm.spec

@@ -0,0 +1,30 @@
+%global armada_folder  /usr/lib/armada
+
+Summary: StarlingX Monitor Application Armada Helm Charts
+Name: stx-monitor-helm
+Version: 1.0
+Release: %{tis_patch_ver}%{?_tis_dist}
+License: Apache-2.0
+Group: base
+Packager: Wind River <info@windriver.com>
+URL: unknown
+
+Source0: %{name}-%{version}.tar.gz
+
+BuildArch: noarch
+BuildRequires: monitor-helm
+Requires: monitor-helm
+
+%description
+StarlingX Monitor Application Armada Helm Charts
+
+%prep
+%setup
+
+%install
+install -d -m 755 ${RPM_BUILD_ROOT}%{armada_folder}
+install -p -D -m 755 manifests/*.yaml ${RPM_BUILD_ROOT}%{armada_folder}
+
+%files
+%defattr(-,root,root,-)
+%{armada_folder}/*

stx-monitor-helm/stx-monitor-helm/Makefile

@@ -0,0 +1,43 @@
+#
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# It's necessary to set this because some environments don't link sh -> bash.
+SHELL := /bin/bash
+TASK  := build
+
+EXCLUDES := helm-toolkit doc tests tools logs tmp
+CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
+
+.PHONY: $(EXCLUDES) $(CHARTS)
+
+all: $(CHARTS)
+
+$(CHARTS):
+	@if [ -d $@ ]; then \
+		echo; \
+		echo "===== Processing [$@] chart ====="; \
+		make $(TASK)-$@; \
+	fi
+
+init-%:
+	if [ -f $*/Makefile ]; then make -C $*; fi
+	if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
+
+lint-%: init-%
+	if [ -d $* ]; then helm lint $*; fi
+
+build-%: lint-%
+	if [ -d $* ]; then helm package $*; fi
+
+clean:
+	@echo "Clean all build artifacts"
+	rm -f */templates/_partials.tpl */templates/_globals.tpl
+	rm -f *tgz */charts/*tgz */requirements.lock
+	rm -rf */charts */tmpcharts
+
+%:
+	@:

stx-monitor-helm/stx-monitor-helm/README

@@ -0,0 +1,5 @@
+This directory contains all StarlingX charts that need to be built for this
+application. Some charts are common across applications. These common charts
+reside in the stx-config/kubernetes/helm-charts directory. To include these in
+this application update the build_srpm.data file and use the COPY_LIST_TO_TAR
+mechanism to populate these common charts.

stx-monitor-helm/stx-monitor-helm/files/index.yaml

@@ -0,0 +1,3 @@
+apiVersion: v1
+entries: {}
+generated: 2019-01-07T12:33:46.098166523-06:00

stx-monitor-helm/stx-monitor-helm/files/repositories.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+generated: 2019-01-02T15:19:36.215111369-06:00
+repositories:
+- caFile: ""
+  cache: /builddir/.helm/repository/cache/local-index.yaml
+  certFile: ""
+  keyFile: ""
+  name: local
+  password: ""
+  url: http://127.0.0.1:8879/charts
+  username: ""
+

stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml

@@ -0,0 +1,573 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: nginx-ingress
+data:
+  chart_name: nginx-ingress
+  release: nginx-ingress
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-nginx-ingress
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-nginx-ingress
+  values:
+    controller:
+      kind: DaemonSet
+      daemonset:
+        useHostPort: false
+      nodeSelector:
+        elastic-controller: "enabled"
+      config:
+        # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
+        nginx-status-ipv4-whitelist: 0.0.0.0/0
+        # See https://bugs.launchpad.net/starlingx/+bug/1823803
+        # Note quotes are necessary.
+        worker-processes: '1'
+      scope:
+        enabled: true
+        namespace: "monitor"
+      service:
+        type: "NodePort"
+        nodePorts:
+          http: 31001
+    defaultBackend:
+      nodeSelector:
+        elastic-controller: "enabled"
+      service:
+        nodePorts:
+          http: 31001
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/nginx-ingress-1.4.0.tgz
+    subpath: nginx-ingress
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: elasticsearch
+data:
+  chart_name: elasticsearch
+  release: elasticsearch
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-elasticsearch
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-elasticsearch
+        - type: pod
+          labels:
+            release: stx-elasticsearch
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+      tag: 7.2.0
+    cluster:
+      config:
+        cluster.initial_master_nodes:
+          - stx-elasticsearch-master-0
+          - stx-elasticsearch-master-1
+      env:
+        MINIMUM_MASTER_NODES: "1"
+        EXPECTED_MASTER_NODES: "1"
+        RECOVER_AFTER_MASTER_NODES: "1"
+    client:
+      ingress:
+        enabled: true
+        annotations:
+          kubernetes.io/ingress.class: "nginx"
+          nginx.ingress.kubernetes.io/rewrite-target: /$2
+          nginx.ingress.kubernetes.io/ssl-redirect: "false"
+          nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+        # Note in order to differentiate from kibana, prefix with
+        # stx-elasticsearch-client, but strip before presenting
+        # to elasticsearch with above rewrite-target
+        path: /stx-elasticsearch-client(/|$)(.*)
+        hosts:
+          - ""
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-1.24.0.tgz
+    subpath: elasticsearch
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: filebeat
+data:
+  chart_name: filebeat
+  release: filebeat
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-filebeat
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-filebeat
+        - type: pod
+          labels:
+            release: stx-filebeat
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/beats/filebeat-oss"
+      tag: 7.2.0
+    config:
+      output.file:
+        enabled: false
+      output.logstash:
+        enabled: true
+        hosts: ["stx-logstash:5044"]
+      output.elasticsearch:
+        enabled: false
+        hosts: ["http://stx-elasticsearch-client:9200"]
+        ilm.pattern: "000001"
+      setup.kibana:
+        #  Note port 5601 is default
+        host: "stx-kibana"
+      setup.dashboards:
+        enabled: true
+      filebeat.autodiscover:
+        providers:
+          - type: kubernetes
+            host: ${NODE_NAME}
+            hints.enabled: true
+    # Mount the setup_script to main container for visibility
+    # even though not used there.
+    extraVolumeMounts:
+      - mountPath: /usr/share/filebeat/setup-script.sh
+        name: setupscript
+        subPath: setup-script.sh
+    extraVolumes:
+      - configMap:
+          defaultMode: 493
+          name: stx-filebeat
+        name: setupscript
+    tolerations:
+      - key: "services"
+        operator: "Equal"
+        value: "disabled"
+        effect: "NoExecute"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/filebeat-1.5.1.tgz
+    subpath: filebeat
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: metricbeat
+data:
+  chart_name: metricbeat
+  release: metricbeat
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-metricbeat
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-metricbeat
+        - type: pod
+          labels:
+            release: stx-metricbeat
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/beats/metricbeat-oss"
+      tag: 7.2.0
+    daemonset:
+      config:
+        output.file:
+          enabled: false
+        output.logstash:
+          enabled: true
+          hosts: ["stx-logstash:5044"]
+        output.elasticsearch:
+          # this must be opposite of above output.logstash.enabled
+          enabled: false
+          hosts: ["http://stx-elasticsearch-client:9200"]
+          ilm.pattern: "000001"
+        processors:
+          - add_kubernetes_metadata:
+              in_cluster: true
+        setup.kibana:
+          # for on box kibana, note port 5601 is default
+          host: "stx-kibana"
+        setup.dashboards:
+          enabled: true
+      metricbeat.autodiscover:
+        providers:
+          - type: kubernetes
+            host: ${NODE_NAME}
+            hints.enabled: true
+      tolerations:
+        - key: "services"
+          operator: "Equal"
+          value: "disabled"
+          effect: "NoExecute"
+    deployment:
+      nodeSelector:
+        elastic-controller: "enabled"
+      config:
+        output.file:
+          enabled: false
+        output.logstash:
+          enabled: true
+          hosts: ["stx-logstash:5044"]
+        output.elasticsearch:
+          enabled: false
+          hosts: ["http://stx-elasticsearch-client:9200"]
+          ilm.pattern: "000001"
+        setup.kibana:
+          # for onbox kibana, note port 5601 is default
+          host: "stx-kibana"
+          # for offbox kibana, will fail if we can't connect
+          # host: "128.224.140.223"
+        setup.dashboards:
+          enabled: true
+        metricbeat.autodiscover:
+          providers:
+            - type: kubernetes
+              host: ${NODE_NAME}
+              hints.enabled: true
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/metricbeat-1.6.0.tgz
+    subpath: metricbeat
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kube-state-metrics
+data:
+  chart_name: kube-state-metrics
+  release: kube-state-metrics
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-kube-state-metrics
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-kube-state-metrics
+        - type: pod
+          labels:
+            release: stx-kube-state-metrics
+            component: test
+  values:
+    image:
+      tag: v1.5.0
+    nodeSelector:
+      elastic-controller: "enabled"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/kube-state-metrics-0.16.0.tgz
+    subpath: kube-state-metrics
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kibana
+data:
+  chart_name: kibana
+  release: kibana
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-kibana
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-kibana
+        - type: pod
+          labels:
+            release: stx-kibana
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/kibana/kibana-oss"
+      tag: 7.2.0
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: "nginx"
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      hosts:
+        - ""
+    files:
+      kibana.yml:
+        elasticsearch.hosts: http://stx-elasticsearch-client:9200
+    nodeSelector:
+      elastic-controller: "enabled"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/kibana-2.2.0.tgz
+    subpath: kibana
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: logstash
+data:
+  chart_name: logstash
+  release: logstash
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-logstash
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-logstash
+        - type: pod
+          labels:
+            release: stx-logstash
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/logstash/logstash-oss"
+      tag: 7.2.0
+    nodeSelector:
+      elastic-controller: "enabled"
+    elasticsearch:
+      host: stx-elasticsearch-client
+    outputs:
+      main: |-
+        output {
+          # Note uncomment below and can see raw input in logs
+          #stdout { codec => rubydebug }
+          if [type] == "collectd" {
+            elasticsearch {
+              hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
+              manage_template => false
+              index => "collectd"
+            }
+          }
+          if [type] == "beats" {
+            elasticsearch {
+              hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
+              manage_template => false
+              index => "%{[@metadata][beat]}-%{[@metadata][version]}"
+            }
+          }
+        }
+    inputs:
+      main: |-
+        input {
+          udp {
+            #Note default port 25826 in use by influxdb.
+            #port => 25826
+            port => 31005
+            buffer_size => 1452
+            codec => collectd { }
+            type => 'collectd'
+          }
+          beats {
+            port => 5044
+            type => 'beats'
+          }
+        }
+    ports:
+      # influxdb port conflict, can't use 25826
+      # - containerPort: 25826
+      # name: collectd-udp
+      # protocol: UDP
+      - containerPort: 31005
+        name: collectd-udp
+        protocol: UDP
+      - containerPort: 5044
+        name: beats
+        protocol: TCP
+    service:
+      ports:
+        # collectd-udp:
+        # port: 25826
+        # targetPort: collectd-udp
+        # protocol: UDP
+        collectd-udp:
+          port: 31005
+          targetPort: collectd-udp
+          protocol: UDP
+        monitor:
+          port: 9600
+          targetPort: monitor
+          protocol: TCP
+
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: "nginx"
+        nginx.ingress.kubernetes.io/rewrite-target: /$2
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      path: /stx-logstash(/|$)(.*)
+      servicePort: 9600
+      hosts:
+        - ""
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/logstash-1.7.0.tgz
+    subpath: logstash
+    reference: master
+  dependencies: []
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: logstash
+data:
+  description: "Deploy logstash"
+  sequenced: true
+  chart_group:
+    - logstash
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: elasticsearch
+data:
+  description: "Deploy elasticsearch"
+  sequenced: true
+  chart_group:
+    - elasticsearch
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: filebeat
+data:
+  description: "Deploy filebeat"
+  sequenced: true
+  chart_group:
+    - filebeat
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: metricbeat
+data:
+  description: "Deploy metricbeat"
+  sequenced: true
+  chart_group:
+    - metricbeat
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kube-state-metrics
+data:
+  description: "Deploy kube-state-metrics"
+  sequenced: true
+  chart_group:
+    - kube-state-metrics
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: nginx-ingress
+data:
+  description: "Deploy ingress"
+  sequenced: true
+  chart_group:
+    - nginx-ingress
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kibana
+data:
+  description: "Deploy kibana"
+  sequenced: true
+  chart_group:
+    - kibana
+---
+schema: armada/Manifest/v1
+metadata:
+  schema: metadata/Document/v1
+  name: monitor-armada-manifest
+data:
+  release_prefix: stx
+  chart_groups:
+    - nginx-ingress
+    - kibana
+    - elasticsearch
+    - logstash
+    - filebeat
+    - metricbeat
+    - kube-state-metrics