Infrastructure and Cluster Monitoring

Introduce a new Armada manifest (stx-monitor) that defines the
necessary software components and services to deploy a
monitoring solution for the infrastructure and Kubernetes cluster
as well as the services it is running.

The Elastic 7.x Apache 2.0 Licensed ("oss") images are included.

The following Elastic components are introduced in this commit:
elasticsearch, filebeat, metricbeat, logstash, kibana.
This enables the build of the stx-monitor application manifest
and upon application, the capturing of logs  and metrics
into elasticsearch.  The logs and metrics may then be viewed with a
tool such as Kibana.

The stx-monitor application is built:
   build-pkgs monitor-helm
   build-pkgs stx-monitor-helm
   build-helm-charts.sh --app stx-monitor

The settings for elasticsearch helm chart are set to allow mtce actions,
such as host-lock, of a controller.

The following host labels are required on controllers prior
to the application-apply:
    system host-label-list controller-0
    (and, in duplex, for controller-1)
    | controller-0 | elastic-controller      | enabled     |
    | controller-0 | elastic-data            | enabled     |
    | controller-0 | elastic-client          | enabled     |

Tests:
    build-helm-charts.sh --app stx-monitor
    system application-upload
    system application-apply
    host management: host-lock, host-swact, host-unlock
    Determine the port for kibana:
        kubectl get service -n monitor
        Access kibana via <oamip>:<service port>
    verify logs and container logs are captured, include subdirs
        under /var/log such as /var/log/openvswitch
    system application-remove
    system application-delete

Change-Id: I0e2f94d302b3ec3a22ee5853d529efb6b423bdaa
Depends-On: Ic9a5b909dfbf60e39f98f3f14fe9bbac2de1d42b
Story: 2005733
Task: 31081
Co-Authored-By: Kevin Smith <kevin.smith@windriver.com>
Signed-off-by: John Kung <john.kung@windriver.com>

stx-monitor-helm/centos/build_srpm.data

@@ -0,0 +1,6 @@
+SRC_DIR="stx-monitor-helm"
+# This version is used as a component of the stx-monitor application
+# version. Any change to this version must also be reflected in the
+# SUPPORTED_VERSIONS list in sysinv/helm/monitor_version_check.py
+#
+TIS_PATCH_VER=1

stx-monitor-helm/centos/stx-monitor-helm.spec

@@ -0,0 +1,30 @@
+%global armada_folder  /usr/lib/armada
+
+Summary: StarlingX Monitor Application Armada Helm Charts
+Name: stx-monitor-helm
+Version: 1.0
+Release: %{tis_patch_ver}%{?_tis_dist}
+License: Apache-2.0
+Group: base
+Packager: Wind River <info@windriver.com>
+URL: unknown
+
+Source0: %{name}-%{version}.tar.gz
+
+BuildArch: noarch
+BuildRequires: monitor-helm
+Requires: monitor-helm
+
+%description
+StarlingX Monitor Application Armada Helm Charts
+
+%prep
+%setup
+
+%install
+install -d -m 755 ${RPM_BUILD_ROOT}%{armada_folder}
+install -p -D -m 755 manifests/*.yaml ${RPM_BUILD_ROOT}%{armada_folder}
+
+%files
+%defattr(-,root,root,-)
+%{armada_folder}/*

stx-monitor-helm/stx-monitor-helm/Makefile

@@ -0,0 +1,43 @@
+#
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# It's necessary to set this because some environments don't link sh -> bash.
+SHELL := /bin/bash
+TASK  := build
+
+EXCLUDES := helm-toolkit doc tests tools logs tmp
+CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
+
+.PHONY: $(EXCLUDES) $(CHARTS)
+
+all: $(CHARTS)
+
+$(CHARTS):
+	@if [ -d $@ ]; then \
+		echo; \
+		echo "===== Processing [$@] chart ====="; \
+		make $(TASK)-$@; \
+	fi
+
+init-%:
+	if [ -f $*/Makefile ]; then make -C $*; fi
+	if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
+
+lint-%: init-%
+	if [ -d $* ]; then helm lint $*; fi
+
+build-%: lint-%
+	if [ -d $* ]; then helm package $*; fi
+
+clean:
+	@echo "Clean all build artifacts"
+	rm -f */templates/_partials.tpl */templates/_globals.tpl
+	rm -f *tgz */charts/*tgz */requirements.lock
+	rm -rf */charts */tmpcharts
+
+%:
+	@:

stx-monitor-helm/stx-monitor-helm/README

@@ -0,0 +1,5 @@
+This directory contains all StarlingX charts that need to be built for this
+application. Some charts are common across applications. These common charts
+reside in the stx-config/kubernetes/helm-charts directory. To include these in
+this application update the build_srpm.data file and use the COPY_LIST_TO_TAR
+mechanism to populate these common charts.

stx-monitor-helm/stx-monitor-helm/files/index.yaml

@@ -0,0 +1,3 @@
+apiVersion: v1
+entries: {}
+generated: 2019-01-07T12:33:46.098166523-06:00

stx-monitor-helm/stx-monitor-helm/files/repositories.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+generated: 2019-01-02T15:19:36.215111369-06:00
+repositories:
+- caFile: ""
+  cache: /builddir/.helm/repository/cache/local-index.yaml
+  certFile: ""
+  keyFile: ""
+  name: local
+  password: ""
+  url: http://127.0.0.1:8879/charts
+  username: ""
+

stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml

@@ -0,0 +1,573 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: nginx-ingress
+data:
+  chart_name: nginx-ingress
+  release: nginx-ingress
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-nginx-ingress
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-nginx-ingress
+  values:
+    controller:
+      kind: DaemonSet
+      daemonset:
+        useHostPort: false
+      nodeSelector:
+        elastic-controller: "enabled"
+      config:
+        # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
+        nginx-status-ipv4-whitelist: 0.0.0.0/0
+        # See https://bugs.launchpad.net/starlingx/+bug/1823803
+        # Note quotes are necessary.
+        worker-processes: '1'
+      scope:
+        enabled: true
+        namespace: "monitor"
+      service:
+        type: "NodePort"
+        nodePorts:
+          http: 31001
+    defaultBackend:
+      nodeSelector:
+        elastic-controller: "enabled"
+      service:
+        nodePorts:
+          http: 31001
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/nginx-ingress-1.4.0.tgz
+    subpath: nginx-ingress
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: elasticsearch
+data:
+  chart_name: elasticsearch
+  release: elasticsearch
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-elasticsearch
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-elasticsearch
+        - type: pod
+          labels:
+            release: stx-elasticsearch
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+      tag: 7.2.0
+    cluster:
+      config:
+        cluster.initial_master_nodes:
+          - stx-elasticsearch-master-0
+          - stx-elasticsearch-master-1
+      env:
+        MINIMUM_MASTER_NODES: "1"
+        EXPECTED_MASTER_NODES: "1"
+        RECOVER_AFTER_MASTER_NODES: "1"
+    client:
+      ingress:
+        enabled: true
+        annotations:
+          kubernetes.io/ingress.class: "nginx"
+          nginx.ingress.kubernetes.io/rewrite-target: /$2
+          nginx.ingress.kubernetes.io/ssl-redirect: "false"
+          nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+        # Note in order to differentiate from kibana, prefix with
+        # stx-elasticsearch-client, but strip before presenting
+        # to elasticsearch with above rewrite-target
+        path: /stx-elasticsearch-client(/|$)(.*)
+        hosts:
+          - ""
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-1.24.0.tgz
+    subpath: elasticsearch
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: filebeat
+data:
+  chart_name: filebeat
+  release: filebeat
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-filebeat
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-filebeat
+        - type: pod
+          labels:
+            release: stx-filebeat
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/beats/filebeat-oss"
+      tag: 7.2.0
+    config:
+      output.file:
+        enabled: false
+      output.logstash:
+        enabled: true
+        hosts: ["stx-logstash:5044"]
+      output.elasticsearch:
+        enabled: false
+        hosts: ["http://stx-elasticsearch-client:9200"]
+        ilm.pattern: "000001"
+      setup.kibana:
+        #  Note port 5601 is default
+        host: "stx-kibana"
+      setup.dashboards:
+        enabled: true
+      filebeat.autodiscover:
+        providers:
+          - type: kubernetes
+            host: ${NODE_NAME}
+            hints.enabled: true
+    # Mount the setup_script to main container for visibility
+    # even though not used there.
+    extraVolumeMounts:
+      - mountPath: /usr/share/filebeat/setup-script.sh
+        name: setupscript
+        subPath: setup-script.sh
+    extraVolumes:
+      - configMap:
+          defaultMode: 493
+          name: stx-filebeat
+        name: setupscript
+    tolerations:
+      - key: "services"
+        operator: "Equal"
+        value: "disabled"
+        effect: "NoExecute"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/filebeat-1.5.1.tgz
+    subpath: filebeat
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: metricbeat
+data:
+  chart_name: metricbeat
+  release: metricbeat
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-metricbeat
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-metricbeat
+        - type: pod
+          labels:
+            release: stx-metricbeat
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/beats/metricbeat-oss"
+      tag: 7.2.0
+    daemonset:
+      config:
+        output.file:
+          enabled: false
+        output.logstash:
+          enabled: true
+          hosts: ["stx-logstash:5044"]
+        output.elasticsearch:
+          # this must be opposite of above output.logstash.enabled
+          enabled: false
+          hosts: ["http://stx-elasticsearch-client:9200"]
+          ilm.pattern: "000001"
+        processors:
+          - add_kubernetes_metadata:
+              in_cluster: true
+        setup.kibana:
+          # for on box kibana, note port 5601 is default
+          host: "stx-kibana"
+        setup.dashboards:
+          enabled: true
+      metricbeat.autodiscover:
+        providers:
+          - type: kubernetes
+            host: ${NODE_NAME}
+            hints.enabled: true
+      tolerations:
+        - key: "services"
+          operator: "Equal"
+          value: "disabled"
+          effect: "NoExecute"
+    deployment:
+      nodeSelector:
+        elastic-controller: "enabled"
+      config:
+        output.file:
+          enabled: false
+        output.logstash:
+          enabled: true
+          hosts: ["stx-logstash:5044"]
+        output.elasticsearch:
+          enabled: false
+          hosts: ["http://stx-elasticsearch-client:9200"]
+          ilm.pattern: "000001"
+        setup.kibana:
+          # for onbox kibana, note port 5601 is default
+          host: "stx-kibana"
+          # for offbox kibana, will fail if we can't connect
+          # host: "128.224.140.223"
+        setup.dashboards:
+          enabled: true
+        metricbeat.autodiscover:
+          providers:
+            - type: kubernetes
+              host: ${NODE_NAME}
+              hints.enabled: true
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/metricbeat-1.6.0.tgz
+    subpath: metricbeat
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kube-state-metrics
+data:
+  chart_name: kube-state-metrics
+  release: kube-state-metrics
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-kube-state-metrics
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-kube-state-metrics
+        - type: pod
+          labels:
+            release: stx-kube-state-metrics
+            component: test
+  values:
+    image:
+      tag: v1.5.0
+    nodeSelector:
+      elastic-controller: "enabled"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/kube-state-metrics-0.16.0.tgz
+    subpath: kube-state-metrics
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kibana
+data:
+  chart_name: kibana
+  release: kibana
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-kibana
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-kibana
+        - type: pod
+          labels:
+            release: stx-kibana
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/kibana/kibana-oss"
+      tag: 7.2.0
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: "nginx"
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      hosts:
+        - ""
+    files:
+      kibana.yml:
+        elasticsearch.hosts: http://stx-elasticsearch-client:9200
+    nodeSelector:
+      elastic-controller: "enabled"
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/kibana-2.2.0.tgz
+    subpath: kibana
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: logstash
+data:
+  chart_name: logstash
+  release: logstash
+  namespace: monitor
+  wait:
+    timeout: 600
+    labels:
+      release: stx-logstash
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release: stx-logstash
+        - type: pod
+          labels:
+            release: stx-logstash
+            component: test
+  values:
+    image:
+      repository: "docker.elastic.co/logstash/logstash-oss"
+      tag: 7.2.0
+    nodeSelector:
+      elastic-controller: "enabled"
+    elasticsearch:
+      host: stx-elasticsearch-client
+    outputs:
+      main: |-
+        output {
+          # Note uncomment below and can see raw input in logs
+          #stdout { codec => rubydebug }
+          if [type] == "collectd" {
+            elasticsearch {
+              hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
+              manage_template => false
+              index => "collectd"
+            }
+          }
+          if [type] == "beats" {
+            elasticsearch {
+              hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
+              manage_template => false
+              index => "%{[@metadata][beat]}-%{[@metadata][version]}"
+            }
+          }
+        }
+    inputs:
+      main: |-
+        input {
+          udp {
+            #Note default port 25826 in use by influxdb.
+            #port => 25826
+            port => 31005
+            buffer_size => 1452
+            codec => collectd { }
+            type => 'collectd'
+          }
+          beats {
+            port => 5044
+            type => 'beats'
+          }
+        }
+    ports:
+      # influxdb port conflict, can't use 25826
+      # - containerPort: 25826
+      # name: collectd-udp
+      # protocol: UDP
+      - containerPort: 31005
+        name: collectd-udp
+        protocol: UDP
+      - containerPort: 5044
+        name: beats
+        protocol: TCP
+    service:
+      ports:
+        # collectd-udp:
+        # port: 25826
+        # targetPort: collectd-udp
+        # protocol: UDP
+        collectd-udp:
+          port: 31005
+          targetPort: collectd-udp
+          protocol: UDP
+        monitor:
+          port: 9600
+          targetPort: monitor
+          protocol: TCP
+
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: "nginx"
+        nginx.ingress.kubernetes.io/rewrite-target: /$2
+        nginx.ingress.kubernetes.io/ssl-redirect: "false"
+        nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      path: /stx-logstash(/|$)(.*)
+      servicePort: 9600
+      hosts:
+        - ""
+  source:
+    type: tar
+    location: http://172.17.0.1:8080/helm_charts/starlingx/logstash-1.7.0.tgz
+    subpath: logstash
+    reference: master
+  dependencies: []
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: logstash
+data:
+  description: "Deploy logstash"
+  sequenced: true
+  chart_group:
+    - logstash
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: elasticsearch
+data:
+  description: "Deploy elasticsearch"
+  sequenced: true
+  chart_group:
+    - elasticsearch
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: filebeat
+data:
+  description: "Deploy filebeat"
+  sequenced: true
+  chart_group:
+    - filebeat
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: metricbeat
+data:
+  description: "Deploy metricbeat"
+  sequenced: true
+  chart_group:
+    - metricbeat
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kube-state-metrics
+data:
+  description: "Deploy kube-state-metrics"
+  sequenced: true
+  chart_group:
+    - kube-state-metrics
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: nginx-ingress
+data:
+  description: "Deploy ingress"
+  sequenced: true
+  chart_group:
+    - nginx-ingress
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: kibana
+data:
+  description: "Deploy kibana"
+  sequenced: true
+  chart_group:
+    - kibana
+---
+schema: armada/Manifest/v1
+metadata:
+  schema: metadata/Document/v1
+  name: monitor-armada-manifest
+data:
+  release_prefix: stx
+  chart_groups:
+    - nginx-ingress
+    - kibana
+    - elasticsearch
+    - logstash
+    - filebeat
+    - metricbeat
+    - kube-state-metrics