openstack/openstack-helm-infra
Code Issues Proposed changes

Elasticsearch S3 Update

Browse Source 
This change updates how the Elasticsearch chart handles
S3 configuration and snapshot repository registration.

This allows for
  - Multiple snapshot destinations to be configued
  - Repositories to use a specific placement target
  - Management of multiple account credentials

Change-Id: I12de918adc5964a4ded46f6f6cd3fa94c7235112
This commit is contained in:
Steven Fitzpatrick 2021-03-15 04:17:44 +00:00
parent 20cf2db961
commit 6de864110e
30 changed files with 374 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elasticsearch/Chart.yaml
View File

@ -12,10 +12,10 @@
---
apiVersion: v1
appVersion: v7.1.0
appVersion: v7.6.2
description: OpenStack-Helm ElasticSearch
name: elasticsearch
version: 0.1.8
version: 0.2.0
home: https://www.elastic.co/
sources:
- https://github.com/elastic/elasticsearch

75
elasticsearch/templates/bin/_create_s3_users.sh.tpl Normal file
View File

@ -0,0 +1,75 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
#!/bin/bash
set -e
function create_s3_user () {
echo "Creating s3 user and key pair"
radosgw-admin user create \
--uid=${S3_USERNAME} \
--display-name=${S3_USERNAME} \
--key-type=s3 \
--access-key ${S3_ACCESS_KEY} \
--secret-key ${S3_SECRET_KEY}
}
function update_s3_user () {
# Retrieve old access keys, if they exist
old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \
| jq -r '.keys[].access_key' || true)
if [[ ! -z ${old_access_keys} ]]; then
for access_key in $old_access_keys; do
# If current access key is the same as the key supplied, do nothing.
if [ "$access_key" == "${S3_ACCESS_KEY}" ]; then
echo "Current user and key pair exists."
continue
else
# If keys differ, remove previous key
radosgw-admin key rm --uid=${S3_USERNAME} --key-type=s3 --access-key=$access_key
fi
done
fi
# Perform one more additional check to account for scenarios where multiple
# key pairs existed previously, but one existing key was the supplied key
current_access_key=$(radosgw-admin user info --uid=${S3_USERNAME} \
| jq -r '.keys[].access_key' || true)
# If the supplied key does not exist, modify the user
if [[ -z ${current_access_key} ]]; then
# Modify user with new access and secret keys
echo "Updating existing user's key pair"
radosgw-admin user modify \
--uid=${S3_USERNAME}\
--access-key ${S3_ACCESS_KEY} \
--secret-key ${S3_SECRET_KEY}
fi
}
{{- range $client, $config := .Values.storage.s3.clients -}}
{{- if $config.create_user | default false }}
S3_USERNAME=${{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper) }}
S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper) }}
S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper) }}
user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true)
if [[ -z ${user_exists} ]]; then
echo "Creating $S3_USERNAME"
create_s3_user > /dev/null 2>&1
else
echo "Updating $S3_USERNAME"
update_s3_user > /dev/null 2>&1
fi
{{- end }}
{{- end }}

13
elasticsearch/templates/bin/_create_template.sh.tpl
View File

@ -1,4 +1,17 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex

16
elasticsearch/templates/bin/_elasticsearch.sh.tpl
View File

@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@ -19,9 +19,17 @@ set -e
COMMAND="${@:-start}"
function initiate_keystore () {
set -ex
bin/elasticsearch-keystore create
echo ${S3_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.access_key
echo ${S3_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.secret_key
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- range $client, $settings := .Values.storage.s3.clients -}}
{{- $access_key := printf "%s_S3_ACCESS_KEY" ( $client | replace "-" "_" | upper) }}
{{- $secret_key := printf "%s_S3_SECRET_KEY" ( $client | replace "-" "_" | upper) }}
echo ${{$access_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key
echo ${{$secret_key}} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key
{{- end }}
{{- end }}
}
function start () {
@ -95,7 +103,7 @@ function start_data_node () {
echo "Disabling Replica Shard Allocation"
curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
"${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
"${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
\"persistent\": {
\"cluster.routing.allocation.enable\": \"primaries\"
}

57
elasticsearch/templates/bin/_helm-tests.sh.tpl
View File

@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@ -29,26 +29,46 @@ function create_test_index () {
' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
if [ "$index_result" == "True" ];
then
echo "PASS: Test index created!";
echo "PASS: Test index created!";
else
echo "FAIL: Test index not created!";
exit 1;
echo "FAIL: Test index not created!";
exit 1;
fi
}
function check_snapshot_repositories () {
{{ range $repository := .Values.conf.elasticsearch.snapshots.repositories }}
repository={{$repository.name}}
repository_search_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_ENDPOINT}/_cat/repositories" | awk '{print $1}' | grep "\<$repository\>")
if [ "$repository_search_result" == "$repository" ]; then
echo "PASS: The snapshot repository $repository exists!"
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
function check_snapshot_repositories_registered () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length)
if [ "$total_hits" -gt 0 ]; then
echo "PASS: $total_hits Snapshot repositories have been registered!"
else
echo "FAIL: The snapshot repository $respository does not exist! Exiting now";
exit 1;
echo "FAIL: No snapshot repositories found! Exiting";
exit 1;
fi
{{ end }}
}
{{ end }}
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
function check_snapshot_repositories_verified () {
repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r "keys | @sh" )
repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
for repository in $repositories; do
error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/${repository}/_verify" | jq -r '.error')
if [ $error == "null" ]; then
echo "PASS: $repository is verified."
else
echo "FAIL: Error for $repository: $(echo $error | jq -r)"
exit 1;
fi
done
}
{{ end }}
{{ if .Values.manifests.job_elasticsearch_templates }}
# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
@ -57,10 +77,10 @@ function check_templates () {
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
if [ "$total_hits" -gt 0 ]; then
echo "PASS: Successful hits on templates!"
echo "PASS: Successful hits on templates!"
else
echo "FAIL: No hits on query for templates! Exiting";
exit 1;
echo "FAIL: No hits on query for templates! Exiting";
exit 1;
fi
}
{{ end }}
@ -74,7 +94,8 @@ function remove_test_index () {
remove_test_index || true
create_test_index
{{ if .Values.conf.elasticsearch.snapshots.enabled }}
check_snapshot_repositories
check_snapshot_repositories_registered
check_snapshot_repositories_verified
{{ end }}
check_templates
remove_test_index

62
elasticsearch/templates/bin/_register-repository.sh.tpl
View File

@ -1,62 +0,0 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{ $envAll := . }}
set -ex
function contains() {
[[ $1 =~ (^|[[:space:]])$2($|[[:space:]]) ]] && return 0 || return 1
}
function register_snapshot_repository() {
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_HOST}/_snapshot/$1" \
-H 'Content-Type: application/json' -d'
{
"type": "s3",
"settings": {
"endpoint": "'"$RGW_HOST"'",
"protocol": "http",
"bucket": "'"$S3_BUCKET"'"
}
}' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
if [ "$result" == "True" ];
then
echo "Snapshot repository $1 created!";
else
echo "Snapshot repository $1 not created!";
exit 1;
fi
}
function verify_snapshot_repository() {
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
}
# Get names of all current snapshot repositories
snapshot_repos=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_HOST}"/_cat/repositories?format=json | jq -r '.[].id')
# Create snapshot repositories if they don't exist
{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }}
if contains "$snapshot_repos" {{$repository.name}}; then
echo "Snapshot repository {{$repository.name}} exists!"
else
register_snapshot_repository {{$repository.name}}
verify_snapshot_repository {{$repository.name}}
fi
{{ end }}

17
elasticsearch/templates/bin/_verify-repositories.sh.tpl
View File

@ -22,6 +22,17 @@ function verify_snapshot_repository() {
-XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
}
{{ range $repository := $envAll.Values.conf.elasticsearch.snapshots.repositories }}
verify_snapshot_repository {{$repository.name}}
{{ end }}
repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
"${ELASTICSEARCH_HOST}/_snapshot" | jq -r 'keys | @sh')
repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
for repository in $repositories; do
error=$(verify_snapshot_repository $repository | jq -r '.error' )
if [ $error == "null" ]; then
echo "$repository is verified."
else
echo "Error for $repository: $(echo $error | jq -r)"
exit 1;
fi
done

4
elasticsearch/templates/configmap-bin-elasticsearch.yaml
View File

@ -31,9 +31,7 @@ data:
create-s3-bucket.sh: |
{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
create-s3-user.sh: |
{{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }}
register-repository.sh: |
{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create_template.sh: |
{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
verify-repositories.sh: |

17
elasticsearch/templates/configmap-etc-elasticsearch.yaml
View File

@ -15,19 +15,22 @@ limitations under the License.
{{- if .Values.manifests.configmap_etc_elasticsearch }}
{{- $envAll := . }}
{{- if empty .Values.endpoints.ceph_object_store.path.default -}}
{{- set .Values.endpoints.ceph_object_store.path "default" .Values.conf.elasticsearch.snapshots.bucket -}}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- range $client, $config := $envAll.Values.storage.s3.clients }}
{{- $settings := $config.settings }}
{{- $endpoint := $settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }}
{{- $_ := set $settings "endpoint" $endpoint }}
{{- $protocol := $settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }}
{{- $_ := set $settings "protocol" $protocol }}
{{- $_:= set $envAll.Values.conf.elasticsearch.config.s3.client $client $settings }}
{{- end -}}
{{- if empty .Values.conf.elasticsearch.config.s3.client.default.endpoint -}}
{{- $radosgw_host := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
{{- set .Values.conf.elasticsearch.config.s3.client.default "endpoint" $radosgw_host -}}
{{- end -}}
{{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}}
{{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}}
{{- set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
{{- $_:= set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}}
{{- end -}}
---
apiVersion: v1
kind: Secret

13
elasticsearch/templates/deployment-client.yaml
View File

@ -162,16 +162,9 @@ spec:
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- if .Values.pod.env.client }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
{{- end }}

13
elasticsearch/templates/deployment-gateway.yaml
View File

@ -115,16 +115,9 @@ spec:
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- if .Values.pod.env.gateway }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }}
{{- end }}

93
elasticsearch/templates/job-register-snapshot-repository.yaml
View File

@ -1,93 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and (.Values.manifests.job_snapshot_repository) (.Values.conf.elasticsearch.snapshots.enabled) }}
{{- $envAll := . }}
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }}
{{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }}
{{ tuple $envAll "snapshot_repository" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: elasticsearch-register-snapshot-repository
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "elasticsearch" "snapshot-repository" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ dict "envAll" $envAll "podName" "elasticsearch-register-snapshot-repository" "containerNames" (list "register-snapshot-repository" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "snapshot_repository" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
initContainers:
{{ tuple $envAll "snapshot_repository" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: register-snapshot-repository
{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "snapshot_repository" "container" "register_snapshot_repository" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: ELASTICSEARCH_USERNAME
valueFrom:
secretKeyRef:
name: {{ $esUserSecret }}
key: ELASTICSEARCH_USERNAME
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $esUserSecret }}
key: ELASTICSEARCH_PASSWORD
- name: ELASTICSEARCH_HOST
value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
- name: S3_BUCKET
value: {{ .Values.conf.elasticsearch.snapshots.bucket | quote }}
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
- name: RGW_HOST
value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
command:
- /tmp/register-repository.sh
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: elasticsearch-bin
mountPath: /tmp/register-repository.sh
subPath: register-repository.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: elasticsearch-bin
configMap:
name: elasticsearch-bin
defaultMode: 0555
{{- end }}

4
elasticsearch/templates/job-s3-bucket.yaml
View File

@ -3,7 +3,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@ -16,4 +16,4 @@ limitations under the License.
{{- $esBucket := .Values.conf.elasticsearch.snapshots.bucket }}
{{- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" "s3Bucket" $esBucket -}}
{{ $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" }}
{{- end }}
{{- end -}}

2
elasticsearch/templates/job-s3-user.yaml
View File

@ -3,7 +3,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,

13
elasticsearch/templates/secret-s3-user.yaml
View File

@ -3,7 +3,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
@ -13,14 +13,5 @@ limitations under the License.
*/}}
{{- if .Values.manifests.secret_s3 }}
{{- $envAll := . }}
{{- $secretName := index $envAll.Values.secrets.rgw.elasticsearch }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
{{- tuple "elasticsearch" $envAll | include "helm-toolkit.snippets.rgw_s3_secret_creds" | indent 2 -}}
{{ include "helm-toolkit.snippets.rgw_s3_secret_creds" . }}
{{- end }}

13
elasticsearch/templates/statefulset-data.yaml
View File

@ -127,18 +127,11 @@ spec:
value: "false"
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
- name: DISCOVERY_SERVICE
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- if .Values.pod.env.data }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
{{- end }}

13
elasticsearch/templates/statefulset-master.yaml
View File

@ -122,16 +122,9 @@ spec:
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: ES_JAVA_OPTS
value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}"
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- if .Values.pod.env.master }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
{{- end }}

102
elasticsearch/values.yaml
View File

@ -103,6 +103,8 @@ dependencies:
services:
- endpoint: internal
service: elasticsearch
jobs:
- elasticsearch-s3-bucket
image_repo_sync:
services:
- endpoint: internal
@ -120,7 +122,7 @@ dependencies:
verify_repositories:
services: null
jobs:
- elasticsearch-register-snapshot-repository
- create-elasticsearch-templates
s3_user:
services:
- endpoint: internal
@ -131,7 +133,7 @@ dependencies:
tests:
services: null
jobs:
- elasticsearch-register-snapshot-repository
- create-elasticsearch-templates
pod:
env:
@ -705,12 +707,7 @@ conf:
network:
host: 0.0.0.0
s3:
client:
default:
# NOTE(srwilkers): This gets configured dynamically via endpoint
# lookups
endpoint: null
protocol: http
client: {}
node:
ingest: ${NODE_INGEST}
master: ${NODE_MASTER}
@ -722,13 +719,6 @@ conf:
logs: /logs
snapshots:
enabled: false
# NOTE(srwilkers): The path for the radosgw s3 endpoint gets populated
# dynamically with this value to ensure the bucket name and s3 compatible
# radosgw endpoint/path match
bucket: elasticsearch_bucket
repositories:
logstash:
name: logstash_snapshots
env:
java_opts:
client: "-Xms256m -Xmx256m"
@ -783,40 +773,7 @@ conf:
min_age: 14d
actions:
delete: {}
- endpoint: _slm/policy/non-security-snapshots
body:
schedule: "0 30 1 * * ?"
name: "<non-security-logs-snapshot-{now/d}>"
repository: logstash_snapshots
config:
indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
ignore_unavailable: true
include_global_state: false
wait_for_completion: true
max_wait: 64800
wait_interval: 30
ignore_empty_list: true
continue_if_exception: true
disable_action: false
retention:
expire_after: 29d
- endpoint: _slm/policy/security-snapshots
body:
schedule: "0 30 1 * * ?"
name: "<security-logs-snapshot-{now/d}>"
repository: logstash_snapshots
config:
indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
ignore_unavailable: true
include_global_state: false
wait_for_completion: true
max_wait: 18000
wait_interval: 30
ignore_empty_list: true
continue_if_exception: true
disable_action: false
retention:
expire_after: 179d
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
@ -900,15 +857,6 @@ endpoints:
ceph_object_store:
name: radosgw
namespace: null
auth:
elasticsearch:
username: elasticsearch
access_key: "elastic_access_key"
secret_key: "elastic_secret_key"
admin:
username: s3_admin
access_key: "admin_access_key"
secret_key: "admin_secret_key"
hosts:
default: ceph-rgw
public: radosgw
@ -963,7 +911,43 @@ storage:
requests:
storage: 1Gi
storage_class: general
s3:
clients: {}
# These values configure the s3 clients section of elasticsearch.yml
# See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html
# default:
# auth:
# # Values under auth are written to the Secret $client-s3-user-secret
# # and the access & secret keys are added to the elasticsearch keystore
# username: elasticsearch
# access_key: "elastic_access_key"
# secret_key: "elastic_secret_key"
# settings:
# # Configure Client Settings here (https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html)
# # endpoint: Defaults to the ceph-rgw endpoint
# # protocol: Defaults to http
# path_style_access: true # Required for ceph-rgw S3 API
# create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
# backup:
# auth:
# username: elasticsearch
# access_key: "backup_access_key"
# secret_key: "backup_secret_key"
# settings:
# endpoint: s3.example.com # Specify your own s3 endpoint (defaults to the ceph_object_store endpoint)
# path_style_access: false
# create_user: false
buckets: {}
# List of buckets to create (if required).
# (The client field references one of the clients defined above)
# - name: elasticsearch-bucket
# client: default
# options: # list of extra options for s3cmd
# - --region="default:osh-infra"
# - name: backup-bucket
# client: backup
# options: # list of extra options for s3cmd
# - --region="default:backup"
manifests:
configmap_bin_curator: false

3
elasticsearch/values_overrides/apparmor.yaml
View File

@ -6,9 +6,6 @@ pod:
master: null
mandatory_access_control:
type: apparmor
elasticsearch-register-snapshot-repository:
register-snapshot-repository: runtime/default
init: runtime/default
elasticsearch-master:
elasticsearch-master: runtime/default
init: runtime/default

2
helm-toolkit/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Helm-Toolkit
name: helm-toolkit
version: 0.2.9
version: 0.2.10
home: https://docs.openstack.org/openstack-helm
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
sources:

10
helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl
View File

@ -77,15 +77,7 @@ spec:
{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
{{- end }}
{{- with $env := dict "s3UserSecret" $s3UserSecret }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }}
{{- end }}
- name: S3_BUCKET
value: {{ $s3Bucket }}
- name: RGW_HOST
value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
- name: RGW_PROTO
value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp

4
helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl
View File

@ -96,9 +96,7 @@ spec:
{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
{{- end }}
{{- with $env := dict "s3UserSecret" $s3UserSecret }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }}
{{- end }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }}
- name: RGW_HOST
value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
volumeMounts:

52
helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl
View File

@ -16,29 +16,51 @@ limitations under the License.
#!/bin/bash
set -e
CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
function check_rgw_s3_bucket () {
echo "Checking if bucket exists"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
}
function create_rgw_s3_bucket () {
echo "Creating bucket"
s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET
}
function modify_bucket_acl () {
echo "Updating bucket ACL"
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
}
ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
{{- $envAll := . }}
{{- range $bucket := .Values.storage.s3.buckets }}
S3_BUCKET={{ $bucket.name }}
S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }}
S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }}
S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }}
S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }}
{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }}
RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }}
RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
{{- end }}
CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
if [ "$RGW_PROTO" = "http" ]; then
CONNECTION_ARGS+=" --no-ssl"
else
CONNECTION_ARGS+=" --no-check-certificate"
fi
ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY"
USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
function check_rgw_s3_bucket () {
s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET
}
function create_rgw_s3_bucket () {
s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET
}
function modify_bucket_acl () {
s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME
}
echo "Creating Bucket $S3_BUCKET at $RGW_HOST"
check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl )
{{- end }}
{{- end }}

19
helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl
View File

@ -13,10 +13,17 @@ limitations under the License.
*/}}
{{- define "helm-toolkit.snippets.rgw_s3_secret_creds" }}
{{- $userClass := index . 0 -}}
{{- $context := index . 1 -}}
{{- $userContext := index $context.Values.endpoints.ceph_object_store.auth $userClass }}
S3_USERNAME: {{ $userContext.username | b64enc }}
S3_ACCESS_KEY: {{ $userContext.access_key | b64enc }}
S3_SECRET_KEY: {{ $userContext.secret_key | b64enc }}
{{- range $client, $config := .Values.storage.s3.clients -}}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }}
type: Opaque
data:
{{- range $key, $value := $config.auth }}
{{ $key | upper }}: {{ $value | toString | b64enc}}
{{- end }}
{{ end }}
{{- end }}

22
helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl
View File

@ -13,20 +13,22 @@ limitations under the License.
*/}}
{{- define "helm-toolkit.snippets.rgw_s3_user_env_vars" }}
{{- $s3UserSecret := .s3UserSecret }}
- name: S3_USERNAME
{{- range $client, $user := .Values.storage.s3.clients }}
{{- $s3secret := printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }}
- name: {{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper) }}
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_USERNAME
- name: S3_ACCESS_KEY
name: {{ $s3secret }}
key: USERNAME
- name: {{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper) }}
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_ACCESS_KEY
- name: S3_SECRET_KEY
name: {{ $s3secret }}
key: ACCESS_KEY
- name: {{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper) }}
valueFrom:
secretKeyRef:
name: {{ $s3UserSecret }}
key: S3_SECRET_KEY
name: {{ $s3secret }}
key: SECRET_KEY
{{- end }}
{{- end }}

1
releasenotes/notes/elasticsearch.yaml
View File

@ -9,4 +9,5 @@ elasticsearch:
- 0.1.6 Fix elasticsearch-master rendering error
- 0.1.7 Pin Java options to specific versions
- 0.1.8 Disable Curator in Gate & Chart Defaults
- 0.2.0 Add more S3 configuration options
...

1
releasenotes/notes/helm-toolkit.yaml
View File

@ -16,4 +16,5 @@ helm-toolkit:
- 0.2.7 Replace brace expansion with more standardized Posix approach
- 0.2.8 Override the expiry of Ingress TLS certificate
- 0.2.9 Jobs; put labels only in the template spec
- 0.2.10 Add more S3 configuration options
...

16
tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
View File

@ -41,11 +41,24 @@ conf:
enabled: false
rgw_s3:
enabled: true
config:
rgw_relaxed_s3_bucket_names: false
rgw_placement_targets:
- name: osh-infra
index_pool: default.rgw.osh-infra.index
data_pool: default.rgw.osh-infra.data
data-extra-pool: default.rgw.osh-infra.non-ec
- name: backup
index_pool: default.rgw.backup.index
data_pool: default.rgw.backup.data
data-extra-pool: default.rgw.backup.non-ec
pod:
replicas:
rgw: 1
manifests:
job_bootstrap: true
job_rgw_placement_targets: true
EOF
helm upgrade --install radosgw-osh-infra ./ceph-rgw \
--namespace=osh-infra \
@ -61,3 +74,6 @@ helm status radosgw-osh-infra
kubectl delete pods -l application=ceph,release_group=radosgw-osh-infra,component=rgw-test --namespace=osh-infra --ignore-not-found
#NOTE: Test Deployment
helm test radosgw-osh-infra --timeout 900
#NOTE: RGW needs to be restarted for placement-targets to become accessible
kubectl delete pods -l application=ceph,component=rgw -n osh-infra

64
tools/deployment/osh-infra-logging/050-elasticsearch.sh
View File

@ -34,6 +34,70 @@ conf:
elasticsearch:
snapshots:
enabled: true
api_objects:
- endpoint: _snapshot/ceph-rgw
body:
type: s3
settings:
client: default
bucket: elasticsearch-bucket
- endpoint: _snapshot/backup
body:
type: s3
settings:
client: backup
bucket: backup-bucket
- endpoint: _slm/policy/rgw-snapshots
body:
schedule: "0 */3 * * * ?"
name: "<snapshot-{now/d}>"
repository: ceph-rgw
config:
indices: ["*"]
retention:
expire_after: 30d
- endpoint: _slm/policy/backup-snapshots
body:
schedule: "0 */3 * * * ?"
name: "<snapshot-{now/d}>"
repository: backup
config:
indices: ["*"]
retention:
expire_after: 180d
storage:
s3:
clients:
# These values configure the s3 clients section of elasticsearch.yml, with access_key and secret_key being saved to the keystore
default:
auth:
username: elasticsearch
access_key: "elastic_access_key"
secret_key: "elastic_secret_key"
settings:
# endpoint: Defaults to the ceph-rgw endpoint
# protocol: Defaults to http
path_style_access: true # Required for ceph-rgw S3 API
create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
backup: # Change this as you'd like
auth:
username: backup
access_key: "backup_access_key"
secret_key: "backup_secret_key"
settings:
endpoint: radosgw.osh-infra.svc.cluster.local # Using the ingress here to test the endpoint override
path_style_access: true
create_user: true
buckets: # List of buckets to create (if required).
- name: elasticsearch-bucket
client: default
options: # list of extra options for s3cmd
- --region="default:osh-infra"
- name: backup-bucket
client: backup
options: # list of extra options for s3cmd
- --region="default:backup"
EOF
: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"}

2
zuul.d/jobs.yaml
View File

@ -151,7 +151,7 @@
- ./tools/deployment/osh-infra-logging/020-ceph.sh
- - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
- ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
- ./tools/deployment/osh-infra-logging/040-ldap.sh
- - ./tools/deployment/osh-infra-logging/040-ldap.sh
- ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
- - ./tools/deployment/osh-infra-logging/060-fluentd.sh
- ./tools/deployment/osh-infra-logging/070-kibana.sh