From bc20c6c8b6de08659be7e469388fe2f6ac231dd1 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 26 Jul 2019 13:01:00 -0500 Subject: [PATCH] Elasticsearch: Add cron job to verify snapshot repositories This adds a cron job to manually verify all snapshot repositories are registered to any active master and data nodes. This is to address scenarios where master and data nodes do not have the desired snapshot repositories registered following node outages or reboots Change-Id: Ie6f42e95c3ca4dc2ec70f2852a2bde11e59ec097 Signed-off-by: Steve Wilkerson --- .../templates/bin/_verify-repositories.sh.tpl | 29 +++++++ .../configmap-bin-elasticsearch.yaml | 2 + .../cron-job-verify-repositories.yaml | 85 +++++++++++++++++++ elasticsearch/values.yaml | 10 +++ .../deployment/multinode/120-elasticsearch.sh | 3 + .../osh-infra-logging/050-elasticsearch.sh | 3 + 6 files changed, 132 insertions(+) create mode 100644 elasticsearch/templates/bin/_verify-repositories.sh.tpl create mode 100644 elasticsearch/templates/cron-job-verify-repositories.yaml diff --git a/elasticsearch/templates/bin/_verify-repositories.sh.tpl b/elasticsearch/templates/bin/_verify-repositories.sh.tpl new file mode 100644 index 000000000..356aae4eb --- /dev/null +++ b/elasticsearch/templates/bin/_verify-repositories.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ $envAll := . }} + +set -ex + +function verify_snapshot_repository() { + curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify" +} + +{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }} + verify_snapshot_repository {{$repository.name}} +{{ end }} diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml index 6627e2d2d..f3012302c 100644 --- a/elasticsearch/templates/configmap-bin-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -40,6 +40,8 @@ data: {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create_template.sh: | {{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + verify-repositories.sh: | +{{ tuple "bin/_verify-repositories.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/elasticsearch/templates/cron-job-verify-repositories.yaml b/elasticsearch/templates/cron-job-verify-repositories.yaml new file mode 100644 index 000000000..a1b8a9731 --- /dev/null +++ b/elasticsearch/templates/cron-job-verify-repositories.yaml @@ -0,0 +1,85 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.cron_verify_repositories) (.Values.conf.elasticsearch.snapshots.enabled) }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "verify-repositories" }} +{{ tuple $envAll "verify_repositories" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: elasticsearch-verify-repositories + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.verify_repositories.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + initContainers: +{{ tuple $envAll "verify_repositories" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: elasticsearch-verify-repositories +{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + command: + - /tmp/verify-repositories.sh + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-bin + mountPath: /tmp/verify-repositories.sh + subPath: verify-repositories.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 +{{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 8dda2b681..854188e35 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -106,6 +106,10 @@ dependencies: jobs: - elasticsearch-s3-bucket - elasticsearch-cluster-wait + verify_repositories: + services: null + jobs: + - elasticsearch-register-snapshot-repository s3_user: services: - endpoint: internal @@ -373,6 +377,11 @@ jobs: snapshot_repository: backoffLimit: 6 activeDeadlineSeconds: 600 + verify_repositories: + cron: "*/30 * * * *" + history: + success: 3 + failed: 1 conf: httpd: | @@ -836,6 +845,7 @@ manifests: configmap_etc_elasticsearch: true configmap_etc_templates: true cron_curator: true + cron_verify_repositories: true deployment_client: true deployment_master: true ingress: true diff --git a/tools/deployment/multinode/120-elasticsearch.sh b/tools/deployment/multinode/120-elasticsearch.sh index c4a7990a6..2f3b45fe2 100755 --- a/tools/deployment/multinode/120-elasticsearch.sh +++ b/tools/deployment/multinode/120-elasticsearch.sh @@ -21,6 +21,9 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +jobs: + verify_repositories: + cron: "*/3 * * * *" pod: replicas: data: 1 diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index a4de23cef..ed5c3dbd4 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -21,6 +21,9 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +jobs: + verify_repositories: + cron: "*/3 * * * *" monitoring: prometheus: enabled: true