Elastic: Move to the elasticsearch charts at elastic/helm-charts

Story: 2005733
Task: 36962

Depends-On: https://review.opendev.org/687360
Change-Id: Idac62e3ebffba9ace2a2e247636c5726fd6f728d
Signed-off-by: Kevin Smith <kevin.smith@windriver.com>

centos_pkg_dirs

@@ -1,2 +1,3 @@
 monitor-helm
+monitor-helm-elastic
 stx-monitor-helm

monitor-helm-elastic/Readme.rst

@@ -0,0 +1,10 @@
+This repo is for
+https://github.com/elastic/helm-charts/tree/master/elasticsearch
+
+Changes to this repo are needed for StarlingX and those changes are
+not yet merged.
+Rather than clone and diverge the repo, the repo is extracted at a particular
+git SHA, and patches are applied on top.
+
+As those patches are merged, the SHA can be updated and
+the local patches removed.

monitor-helm-elastic/centos/build_srpm.data

@@ -0,0 +1,8 @@
+TAR_NAME=helm-charts-elastic
+SHA=2bd7616ceddbdf2eee88965e2028ee37d304c79c
+VERSION=1.0.0
+TAR="$TAR_NAME-$SHA.tar.gz"
+
+COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/files/* "
+
+TIS_PATCH_VER=1

monitor-helm-elastic/centos/monitor-helm-elastic.spec

@@ -0,0 +1,75 @@
+%global sha 2bd7616ceddbdf2eee88965e2028ee37d304c79c
+%global helm_folder  /usr/lib/helm
+%global helmchart_version 0.1.0
+%global _default_patch_flags --no-backup-if-mismatch --prefix=/tmp/junk
+
+Summary: Monitor-Helm-Elastic charts
+Name: monitor-helm-elastic
+Version: 1.0
+Release: %{tis_patch_ver}%{?_tis_dist}
+License: Apache-2.0
+Group: base
+Packager: Wind River <info@windriver.com>
+URL: https://github.com/elastic/helm-charts/
+
+Source0: helm-charts-elastic-%{sha}.tar.gz
+Source1: repositories.yaml
+Source2: index.yaml
+
+BuildArch:     noarch
+
+Patch01: 0001-add-makefile.patch
+Patch02: 0002-Add-compatibility-for-k8s-1.16.patch
+Patch03: 0003-use-oss-image.patch
+
+BuildRequires: helm
+
+%description
+Monitor Helm elasticsearch charts
+
+%prep
+%setup -n helm-charts-elastic
+%patch01 -p1
+%patch02 -p1
+%patch03 -p1
+
+
+%build
+# initialize helm and build the toolkit
+# helm init --client-only does not work if there is no networking
+# The following commands do essentially the same as: helm init
+%define helm_home %{getenv:HOME}/.helm
+mkdir %{helm_home}
+mkdir %{helm_home}/repository
+mkdir %{helm_home}/repository/cache
+mkdir %{helm_home}/repository/local
+mkdir %{helm_home}/plugins
+mkdir %{helm_home}/starters
+mkdir %{helm_home}/cache
+mkdir %{helm_home}/cache/archive
+
+# Stage a repository file that only has a local repo
+cp %{SOURCE1} %{helm_home}/repository/repositories.yaml
+
+# Stage a local repo index that can be updated by the build
+cp %{SOURCE2} %{helm_home}/repository/local/index.yaml
+
+# Host a server for the charts
+helm serve --repo-path . &
+helm repo rm local
+helm repo add local http://localhost:8879/charts
+
+# Create the tgz files
+rm elasticsearch/Makefile
+make elasticsearch
+
+# terminate helm server (the last backgrounded task)
+kill %1
+
+%install
+install -d -m 755 ${RPM_BUILD_ROOT}%{helm_folder}
+install -p -D -m 755 *.tgz ${RPM_BUILD_ROOT}%{helm_folder}
+
+%files
+%defattr(-,root,root,-)
+%{helm_folder}/*

monitor-helm-elastic/files/0001-add-makefile.patch

@@ -0,0 +1,62 @@
+From c9c3e8ff214360eb2e5d9e7728b7bee8fe771eea Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Tue, 1 Oct 2019 15:25:32 -0400
+Subject: [PATCH 1/1] add makefile
+
+---
+ Makefile | 43 +++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 43 insertions(+)
+ create mode 100644 Makefile
+
+diff --git a/Makefile b/Makefile
+new file mode 100644
+index 0000000..5cf4447
+--- /dev/null
++++ b/Makefile
+@@ -0,0 +1,43 @@
++#
++# Copyright 2017 The Openstack-Helm Authors.
++#
++# Copyright (c) 2018 Wind River Systems, Inc.
++#
++# SPDX-License-Identifier: Apache-2.0
++#
++# It's necessary to set this because some environments don't link sh -> bash.
++SHELL := /bin/bash
++TASK  := build
++
++EXCLUDES := helm-toolkit doc tests tools logs tmp
++CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
++
++.PHONY: $(EXCLUDES) $(CHARTS)
++
++all: $(CHARTS)
++
++$(CHARTS):
++	@if [ -d $@ ]; then \
++		echo; \
++		echo "===== Processing [$@] chart ====="; \
++		make $(TASK)-$@; \
++	fi
++
++init-%:
++	if [ -f $*/Makefile ]; then make -C $*; fi
++	if [ -f $*/requirements.yaml ]; then helm dep up $*; fi
++
++lint-%: init-%
++	if [ -d $* ]; then helm lint $*; fi
++
++build-%: lint-%
++	if [ -d $* ]; then helm package $*; fi
++
++clean:
++	@echo "Clean all build artifacts"
++	rm -f */templates/_partials.tpl */templates/_globals.tpl
++	rm -f *tgz */charts/*tgz */requirements.lock
++	rm -rf */charts */tmpcharts
++
++%:
++	@:
+-- 
+1.8.3.1
+

monitor-helm-elastic/files/0002-Add-compatibility-for-k8s-1.16.patch

@@ -0,0 +1,68 @@
+From f79169b8e6a57881952ca4d2c338eeda179471e9 Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Mon, 7 Oct 2019 11:18:05 -0400
+Subject: [PATCH 1/1] Add compatibility for k8s 1.16
+
+---
+ elasticsearch/templates/_helpers.tpl     | 22 ++++++++++++++++++++++
+ elasticsearch/templates/ingress.yaml     |  2 +-
+ elasticsearch/templates/statefulset.yaml |  2 +-
+ 3 files changed, 24 insertions(+), 2 deletions(-)
+
+diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl
+index d651d56..ccfd8c3 100755
+--- a/elasticsearch/templates/_helpers.tpl
++++ b/elasticsearch/templates/_helpers.tpl
+@@ -47,3 +47,25 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
+   {{- end -}}
+ {{- end -}}
+ {{- end -}}
++
++{{/*
++Return the appropriate apiVersion for statefulset.
++*/}}
++{{- define "elasticsearch.statefulset.apiVersion" -}}
++{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}}
++{{- print "apps/v1beta2" -}}
++{{- else -}}
++{{- print "apps/v1" -}}
++{{- end -}}
++{{- end -}}
++
++{{/*
++Return the appropriate apiVersion for ingress.
++*/}}
++{{- define "elasticsearch.ingress.apiVersion" -}}
++{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}}
++{{- print "extensions/v1beta1" -}}
++{{- else -}}
++{{- print "networking.k8s.io/v1beta1" -}}
++{{- end -}}
++{{- end -}}
+diff --git a/elasticsearch/templates/ingress.yaml b/elasticsearch/templates/ingress.yaml
+index 39167ca..1715b97 100644
+--- a/elasticsearch/templates/ingress.yaml
++++ b/elasticsearch/templates/ingress.yaml
+@@ -2,7 +2,7 @@
+ {{- $fullName := include "uname" . -}}
+ {{- $servicePort := .Values.httpPort -}}
+ {{- $ingressPath := .Values.ingress.path -}}
+-apiVersion: extensions/v1beta1
++apiVersion: {{ template "elasticsearch.ingress.apiVersion" . }}
+ kind: Ingress
+ metadata:
+   name: {{ $fullName }}
+diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml
+index 004578f..36c7af6 100644
+--- a/elasticsearch/templates/statefulset.yaml
++++ b/elasticsearch/templates/statefulset.yaml
+@@ -1,5 +1,5 @@
+ ---
+-apiVersion: apps/v1beta1
++apiVersion: {{ template "elasticsearch.statefulset.apiVersion" . }}
+ kind: StatefulSet
+ metadata:
+   name: {{ template "uname" . }}
+-- 
+1.8.3.1
+

monitor-helm-elastic/files/0003-use-oss-image.patch

@@ -0,0 +1,25 @@
+From a1ea0a2f488660f9cfbab44a4fb8b163e135ebf5 Mon Sep 17 00:00:00 2001
+From: Kevin Smith <kevin.smith@windriver.com>
+Date: Tue, 8 Oct 2019 13:35:57 -0400
+Subject: [PATCH 1/1] use oss image
+
+---
+ elasticsearch/values.yaml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml
+index ccab462..f4fca1f 100755
+--- a/elasticsearch/values.yaml
++++ b/elasticsearch/values.yaml
+@@ -42,7 +42,7 @@ secretMounts: []
+ #    secretName: elastic-certificates
+ #    path: /usr/share/elasticsearch/config/certs
+ 
+-image: "docker.elastic.co/elasticsearch/elasticsearch"
++image: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+ imageTag: "7.3.2"
+ imagePullPolicy: "IfNotPresent"
+ 
+-- 
+1.8.3.1
+

monitor-helm-elastic/files/index.yaml

@@ -0,0 +1,4 @@
+---
+apiVersion: v1
+entries: {}
+generated: 2019-01-07T12:33:46.098166523-06:00

monitor-helm-elastic/files/repositories.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+generated: 2019-01-02T15:19:36.215111369-06:00
+repositories:
+  - caFile: ""
+    cache: /builddir/.helm/repository/cache/local-index.yaml
+    certFile: ""
+    keyFile: ""
+    name: local
+    password: ""
+    url: http://127.0.0.1:8879/charts
+    username: ""

monitor-helm/Readme.rst

@@ -1,5 +1,4 @@
 This repo is for
-https://github.com/helm/charts/tree/master/stable/elasticsearch
 https://github.com/helm/charts/tree/master/stable/filebeat
 https://github.com/helm/charts/tree/master/stable/metricbeat
 https://github.com/helm/charts/tree/master/stable/kibana

monitor-helm/centos/build_srpm.data

@@ -5,4 +5,4 @@ TAR="$TAR_NAME-$SHA.tar.gz"
 
 COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/files/* "
 
-TIS_PATCH_VER=2
+TIS_PATCH_VER=3

monitor-helm/centos/monitor-helm.spec

@@ -70,7 +70,6 @@ helm repo add local http://localhost:8879/charts
 
 # Create the tgz files
 cd stable
-make elasticsearch
 make filebeat
 make metricbeat
 make kube-state-metrics

stx-monitor-helm/centos/stx-monitor-helm.spec

@@ -13,7 +13,9 @@ Source0: %{name}-%{version}.tar.gz
 
 BuildArch: noarch
 BuildRequires: monitor-helm
+BuildRequires: monitor-helm-elastic
 Requires: monitor-helm
+Requires: monitor-helm-elastic
 
 %description
 StarlingX Monitor Application Armada Helm Charts

stx-monitor-helm/stx-monitor-helm/manifests/monitor_manifest.yaml

@@ -54,68 +54,158 @@ data:
   dependencies: []
 ---
 schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
 data:
-  chart_name: elasticsearch
+  chart_name: elasticsearch-data
-  release: elasticsearch
+  dependencies: []
-  namespace: monitor
-  wait:
-    timeout: 600
-    labels:
-      release: mon-elasticsearch
-  test:
-    enabled: false
   install:
     no_hooks: false
+  namespace: monitor
+  release: elasticsearch-data
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
   upgrade:
     no_hooks: false
     pre:
       delete:
-        - type: job
+      - labels:
-          labels:
+          release: mon-elasticsearch-data
-            release: mon-elasticsearch
+        type: job
-        - type: pod
+      - labels:
-          labels:
+          component: test
-            release: mon-elasticsearch
+          release: mon-elasticsearch-data
-            component: test
+        type: pod
   values:
-    image:
+    clusterName: 'mon-elasticsearch'
-      repository: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+    nodeGroup: 'data'
-      tag: 7.2.0
+    roles:
-    cluster:
+      master: 'false'
-      env:
+      data: 'true'
-        MINIMUM_MASTER_NODES: "1"
+      ingest: 'false'
-        EXPECTED_MASTER_NODES: "1"
+    replicas: 2
-        RECOVER_AFTER_MASTER_NODES: "1"
+    minimumMasterNodes: 1
-    client:
+    nodeSelector:
-      ingress:
+      elastic-data: enabled
-        enabled: true
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
-        annotations:
+    imageTag: 7.3.2
-          kubernetes.io/ingress.class: "nginx"
+    esMajorVersion: 7
-          nginx.ingress.kubernetes.io/rewrite-target: /$2
+  wait:
-          nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    labels:
-          nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      release: mon-elasticsearch-data
-        # Note in order to differentiate from kibana, prefix with
+    timeout: 600
-        # mon-elasticsearch-client, but strip before presenting
+metadata:
-        # to elasticsearch with above rewrite-target
+  name: elasticsearch-data
-        path: /mon-elasticsearch-client(/|$)(.*)
+  schema: metadata/Document/v1
-        hosts:
+---
-          - ""
+schema: armada/Chart/v1
-    master:
+data:
-      updateStrategy:
+  chart_name: elasticsearch-client
-        type: "RollingUpdate"
-    data:
-      terminationGracePeriodSeconds: 240
-      updateStrategy:
-        type: "RollingUpdate"
-  source:
-    type: tar
-    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-1.24.0.tgz
-    subpath: elasticsearch
-    reference: master
   dependencies: []
+  install:
+    no_hooks: false
+  namespace: monitor
+  release: elasticsearch-client
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+      - labels:
+          release: mon-elasticsearch-client
+        type: job
+      - labels:
+          component: test
+          release: mon-elasticsearch-client
+        type: pod
+  values:
+    clusterName: 'mon-elasticsearch'
+    nodeGroup: 'client'
+    roles:
+      master: 'false'
+      data: 'false'
+      ingest: 'true'
+    replicas: 2
+    minimumMasterNodes: 1
+    ingress:
+      enabled: true
+      annotations:
+        kubernetes.io/ingress.class: nginx
+        nginx.ingress.kubernetes.io/force-ssl-redirect: 'false'
+        nginx.ingress.kubernetes.io/rewrite-target: /$2
+        nginx.ingress.kubernetes.io/ssl-redirect: 'false'
+      hosts:
+      - ''
+      path: /mon-elasticsearch-client(/|$)(.*)
+    nodeSelector:
+      elastic-client: enabled
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
+    imageTag: 7.3.2
+    esMajorVersion: 7
+  wait:
+    labels:
+      release: mon-elasticsearch-client
+    timeout: 600
+metadata:
+  name: elasticsearch-client
+  schema: metadata/Document/v1
+---
+schema: armada/Chart/v1
+data:
+  chart_name: elasticsearch-master
+  dependencies: []
+  install:
+    no_hooks: false
+  namespace: monitor
+  release: elasticsearch-master
+  source:
+    location: http://172.17.0.1:8080/helm_charts/starlingx/elasticsearch-7.3.2.tgz
+    reference: master
+    subpath: elasticsearch
+    type: tar
+  test:
+    enabled: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+      - labels:
+          release: mon-elasticsearch-master
+        type: job
+      - labels:
+          component: test
+          release: mon-elasticsearch-master
+        type: pod
+  values:
+    clusterName: 'mon-elasticsearch'
+    nodeGroup: 'master'
+    roles:
+      master: 'true'
+      data: 'false'
+      ingest: 'false'
+    replicas: 1
+    minimumMasterNodes: 1
+    nodeSelector:
+      elastic-master: enabled
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss
+    imageTag: 7.3.2
+    esMajorVersion: 7
+  wait:
+    labels:
+      release: mon-elasticsearch-master
+    timeout: 600
+metadata:
+  name: elasticsearch-master
+  schema: metadata/Document/v1
 ---
 schema: armada/Chart/v1
 metadata:
@@ -147,7 +237,7 @@ data:
   values:
     image:
       repository: "docker.elastic.co/beats/filebeat-oss"
-      tag: 7.2.0
+      tag: 7.3.2
     config:
       output.file:
         enabled: false
@@ -225,7 +315,7 @@ data:
   values:
     image:
       repository: "docker.elastic.co/beats/metricbeat-oss"
-      tag: 7.2.0
+      tag: 7.3.2
     daemonset:
       config:
         output.file:
@@ -365,7 +455,7 @@ data:
   values:
     image:
       repository: "docker.elastic.co/kibana/kibana-oss"
-      tag: 7.2.0
+      tag: 7.3.2
     ingress:
       enabled: true
       annotations:
@@ -418,7 +508,7 @@ data:
   values:
     image:
       repository: "docker.elastic.co/logstash/logstash-oss"
-      tag: 7.2.0
+      tag: 7.3.2
     nodeSelector:
       elastic-controller: "enabled"
     elasticsearch:
@@ -518,13 +608,15 @@ data:
 ---
 schema: armada/ChartGroup/v1
 metadata:
-  schema: metadata/Document/v1
   name: elasticsearch
+  schema: metadata/Document/v1
 data:
+  chart_group:
+  - elasticsearch-master
+  - elasticsearch-data
+  - elasticsearch-client
   description: "Deploy elasticsearch"
   sequenced: true
-  chart_group:
-    - elasticsearch
 ---
 schema: armada/ChartGroup/v1
 metadata: