openstack/openstack-helm-infra
Code Issues Proposed changes

Elasticsearch: Make templates job more robust

Browse Source 
This change primarily changes the type of the api_objects yaml structure
to a map, which allows for additional objects to be added by values
overrides (Arrays/Lists are not mutable like this)

Also, in the previous change, some scripts in HTK were modified, while
other were copied over to the Elasticsearch chart. To simplify the chart's
structure, this change also moves the create_s3_bucket script to Elasticsearch,
and reverts the changes in HTK.

Those HTK scripts are no longer referenced by osh charts, and could be candidates
for removal if that chart needed to be pruned

Change-Id: I7d8d7ef28223948437450dcb64bd03f2975ad54d
This commit is contained in:
Steven Fitzpatrick 2021-04-06 17:58:21 +00:00
parent 6de864110e
commit d3c6069be3
12 changed files with 159 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
elasticsearch/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v7.6.2
description: OpenStack-Helm ElasticSearch
name: elasticsearch
version: 0.2.0
version: 0.2.1
home: https://www.elastic.co/
sources:
- https://github.com/elastic/elasticsearch

63
elasticsearch/templates/bin/_create_s3_buckets.sh.tpl Normal file
View File

@ -0,0 +1,63 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
#!/bin/bash
set -e
function check_rgw_s3_bucket () {
echo "Checking if bucket exists"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
}
function create_rgw_s3_bucket () {
echo "Creating bucket"
s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
}
function modify_bucket_acl () {
echo "Updating bucket ACL"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
}
ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
{{- $envAll := . }}
{{- range $bucket := .Values.storage.s3.buckets }}
S3_BUCKET={{ $bucket.name }}
S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }}
RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }}
{{- end }}
CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
if [ "$RGW_PROTO" = "http" ]; then
CONNECTION_ARGS+=" --no-ssl"
fi
USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
{{- end }}

28
elasticsearch/templates/bin/_create_template.sh.tpl
View File

@ -13,11 +13,31 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
set -e
{{ range $object := .Values.conf.api_objects }}
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
NUM_ERRORS=0
{{ range $name, $object := .Values.conf.api_objects }}
{{ if not (empty $object) }}
echo "creating {{$name}}"
error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-X{{ $object.method | default "PUT" | upper }} \
"${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
-H 'Content-Type: application/json' -d '{{ $object.body | toJson }}'
-H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error')
if [ $error == "null" ]; then
echo "Object {{$name}} was created."
else
echo "Error when creating object {{$name}}: $(echo $error | jq -r)"
NUM_ERRORS=$(($NUM_ERRORS+1))
fi
{{ end }}
{{ end }}
if [ $NUM_ERRORS -gt 0 ]; then
exit 1
else
echo "leaving normally"
fi

49
elasticsearch/templates/bin/_helm-tests.sh.tpl
View File

@ -36,17 +36,30 @@ function create_test_index () {
fi
}
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
function check_snapshot_repositories_registered () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length)
if [ "$total_hits" -gt 0 ]; then
echo "PASS: $total_hits Snapshot repositories have been registered!"
{{ if not (empty .Values.conf.api_objects) }}
function test_api_object_creation () {
NUM_ERRORS=0
{{ range $object, $config := .Values.conf.api_objects }}
error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/{{ $config.endpoint }}" | jq -r '.error')
if [ $error == "null" ]; then
echo "PASS: {{ $object }} is verified."
else
echo "FAIL: Error for {{ $object }}: $(echo $error | jq -r)"
NUM_ERRORS=$(($NUM_ERRORS+1))
fi
{{ end }}
if [ $NUM_ERRORS -gt 0 ]; then
echo "FAIL: Some API Objects were not created!"
exit 1
else
echo "FAIL: No snapshot repositories found! Exiting";
exit 1;
echo "PASS: API Objects are verified!"
fi
}
{{ end }}
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
@ -70,21 +83,6 @@ function check_snapshot_repositories_verified () {
}
{{ end }}
{{ if .Values.manifests.job_elasticsearch_templates }}
# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
# templates defined by values.yaml
function check_templates () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
if [ "$total_hits" -gt 0 ]; then
echo "PASS: Successful hits on templates!"
else
echo "FAIL: No hits on query for templates! Exiting";
exit 1;
fi
}
{{ end }}
function remove_test_index () {
echo "Deleting index created for service testing"
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
@ -93,9 +91,8 @@ function remove_test_index () {
remove_test_index || true
create_test_index
remove_test_index
test_api_object_creation
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
check_snapshot_repositories_registered
check_snapshot_repositories_verified
{{ end }}
check_templates
remove_test_index

2
elasticsearch/templates/configmap-bin-elasticsearch.yaml
View File

@ -29,7 +29,7 @@ data:
ceph-admin-keyring.sh: |
{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create-s3-bucket.sh: |
{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
{{ tuple "bin/_create_s3_buckets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create-s3-user.sh: |
{{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create_template.sh: |

54
elasticsearch/values.yaml
View File

@ -736,43 +736,23 @@ conf:
ca: null
client_private_key: null
client_cert: null
api_objects:
- endpoint: _template/fluent
body:
index_patterns: "logstash-*"
settings:
index:
number_of_shards: 1
mappings:
properties:
kubernetes:
properties:
container_name:
type: keyword
index: false
docker_id:
type: keyword
index: false
host:
type: keyword
index: false
namespace_name:
type: keyword
index: false
pod_id:
type: keyword
index: false
pod_name:
type: keyword
index: false
- endpoint: _ilm/policy/delete_all_indexes
body:
policy:
phases:
delete:
min_age: 14d
actions:
delete: {}
api_objects: {}
# Fill this map with API objects to create once Elasticsearch is deployed
# name: # This name can be completely arbitrary
# method: # Defaults to PUT
# endpoint: # Path for the request
# body: # Body of the request in yaml (Converted to Json in Template)
# Example: ILM Policy
# ilm_policy:
# endpoint: _ilm/policy/delete_all_indexes
# body:
# policy:
# phases:
# delete:
# min_age: 14d
# actions:
# delete: {}
endpoints:
cluster_domain_suffix: cluster.local

2
helm-toolkit/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Helm-Toolkit
name: helm-toolkit
version: 0.2.10
version: 0.2.11
home: https://docs.openstack.org/openstack-helm
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
sources:

53
helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl
View File

@ -11,56 +11,25 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.scripts.create_s3_bucket" }}
#!/bin/bash
set -e
function check_rgw_s3_bucket () {
echo "Checking if bucket exists"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
}
function create_rgw_s3_bucket () {
echo "Creating bucket"
s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
}
function modify_bucket_acl () {
echo "Updating bucket ACL"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
}
ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
{{- $envAll := . }}
{{- range $bucket := .Values.storage.s3.buckets }}
S3_BUCKET={{ $bucket.name }}
S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }}
RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
{{- end }}
CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
if [ "$RGW_PROTO" = "http" ]; then
CONNECTION_ARGS+=" --no-ssl"
else
CONNECTION_ARGS+=" --no-check-certificate"
fi
ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
function check_rgw_s3_bucket () {
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
}
function create_rgw_s3_bucket () {
s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET
}
function modify_bucket_acl () {
s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
}
check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
{{- end }}
{{- end }}
{{- end }}

8
helm-toolkit/templates/scripts/_create-s3-user.sh.tpl
View File

@ -11,12 +11,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.scripts.create_s3_user" }}
#!/bin/bash
set -e
function create_s3_user () {
echo "Creating s3 user and key pair"
radosgw-admin user create \
@ -26,7 +23,6 @@ function create_s3_user () {
--access-key ${S3_ACCESS_KEY} \
--secret-key ${S3_SECRET_KEY}
}
function update_s3_user () {
# Retrieve old access keys, if they exist
old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \
@ -60,12 +56,10 @@ function update_s3_user () {
--secret-key ${S3_SECRET_KEY}
fi
}
user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true)
if [[ -z ${user_exists} ]]; then
create_s3_user
else
update_s3_user
fi
{{- end }}
{{- end }}

1
releasenotes/notes/elasticsearch.yaml
View File

@ -10,4 +10,5 @@ elasticsearch:
- 0.1.7 Pin Java options to specific versions
- 0.1.8 Disable Curator in Gate & Chart Defaults
- 0.2.0 Add more S3 configuration options
- 0.2.1 Make templates job more robust & allow overrides
...

1
releasenotes/notes/helm-toolkit.yaml
View File

@ -17,4 +17,5 @@ helm-toolkit:
- 0.2.8 Override the expiry of Ingress TLS certificate
- 0.2.9 Jobs; put labels only in the template spec
- 0.2.10 Add more S3 configuration options
- 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9
...

32
tools/deployment/osh-infra-logging/050-elasticsearch.sh
View File

@ -35,36 +35,34 @@ conf:
snapshots:
enabled: true
api_objects:
- endpoint: _snapshot/ceph-rgw
snapshot_repo:
endpoint: _snapshot/ceph-rgw
body:
type: s3
settings:
client: default
bucket: elasticsearch-bucket
- endpoint: _snapshot/backup
body:
type: s3
settings:
client: backup
bucket: backup-bucket
- endpoint: _slm/policy/rgw-snapshots
slm_policy:
endpoint: _slm/policy/snapshots
body:
schedule: "0 */3 * * * ?"
name: "<snapshot-{now/d}>"
repository: ceph-rgw
config:
indices: ["*"]
indices:
- "<*-{now/d}>"
retention:
expire_after: 30d
- endpoint: _slm/policy/backup-snapshots
ilm_policy:
endpoint: _ilm/policy/cleanup
body:
schedule: "0 */3 * * * ?"
name: "<snapshot-{now/d}>"
repository: backup
config:
indices: ["*"]
retention:
expire_after: 180d
policy:
phases:
delete:
min_age: 5d
actions:
delete: {}
test_empty: {}
storage:
s3:
clients: