From 7f783dba51295085dc67ab97f4b96a09a1e961ee Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Thu, 9 Nov 2023 13:16:49 -0600 Subject: [PATCH] Update elasticsearch chart to work with Rook Ceph When using Rook for managing Ceph we can use Rook CRDs to create S3 buckets and users. This PR adds bucket claim template to the elasticsearch chart. Rook creates a bucket for a bucket claim and also creates a secret containing the credentials to get access to this bucket. So we also add a snippet to expose these credentials via environment variables to containers where they are needed. Change-Id: Ic5cd35a5c64a914af97d2b3cfec21dbe399c0f14 --- elasticsearch/Chart.yaml | 2 +- .../templates/deployment-client.yaml | 4 + .../templates/deployment-gateway.yaml | 4 + .../templates/object-bucket-claim.yaml | 29 ++++++ elasticsearch/templates/statefulset-data.yaml | 4 + .../templates/statefulset-master.yaml | 4 + elasticsearch/values.yaml | 1 + helm-toolkit/Chart.yaml | 2 +- .../_rgw_s3_bucket_user_env_vars_rook.tpl | 28 ++++++ releasenotes/notes/elasticsearch.yaml | 1 + releasenotes/notes/helm-toolkit.yaml | 1 + roles/deploy-env/tasks/control-plane.yaml | 2 +- tools/deployment/ceph/ceph-rook.sh | 28 +++++- .../osh-infra-logging/050-elasticsearch.sh | 91 ++++++++++++++++--- zuul.d/jobs.yaml | 5 +- zuul.d/project.yaml | 10 +- 16 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 elasticsearch/templates/object-bucket-claim.yaml create mode 100644 helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 18e279be5..83e4f6b28 100644 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v8.9.0 description: OpenStack-Helm ElasticSearch name: elasticsearch -version: 0.2.27 +version: 0.2.28 home: https://www.elastic.co/ sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index eb4d4a704..418597519 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -177,8 +177,12 @@ spec: key: ELASTICSEARCH_PASSWORD {{- end }} {{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} {{- end }} +{{- end }} {{- if .Values.pod.env.client }} {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }} {{- end }} diff --git a/elasticsearch/templates/deployment-gateway.yaml b/elasticsearch/templates/deployment-gateway.yaml index 6354fdbb2..f11b1459a 100644 --- a/elasticsearch/templates/deployment-gateway.yaml +++ b/elasticsearch/templates/deployment-gateway.yaml @@ -119,8 +119,12 @@ spec: key: ELASTICSEARCH_PASSWORD {{- end }} {{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} {{- end }} +{{- end }} {{- if .Values.pod.env.gateway }} {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }} {{- end }} diff --git a/elasticsearch/templates/object-bucket-claim.yaml b/elasticsearch/templates/object-bucket-claim.yaml new file mode 100644 index 000000000..a68decb9e --- /dev/null +++ b/elasticsearch/templates/object-bucket-claim.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.object_bucket_claim) (.Values.conf.elasticsearch.snapshots.enabled) }} +{{- range $bucket := .Values.storage.s3.buckets }} +# When using this Rook CRD, not only bucket will be created, +# but also a secret containing the credentials to access the bucket. +--- +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: {{ $bucket.name }} +spec: + bucketName: {{ $bucket.name }} + storageClassName: {{ $bucket.storage_class }} +... +{{- end -}} +{{- end -}} diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml index beb128546..2f95a6080 100644 --- a/elasticsearch/templates/statefulset-data.yaml +++ b/elasticsearch/templates/statefulset-data.yaml @@ -133,8 +133,12 @@ spec: - name: DISCOVERY_SERVICE value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} {{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} {{- end }} +{{- end }} {{- if .Values.pod.env.data }} {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }} {{- end }} diff --git a/elasticsearch/templates/statefulset-master.yaml b/elasticsearch/templates/statefulset-master.yaml index 4833a8411..c9efbef9c 100644 --- a/elasticsearch/templates/statefulset-master.yaml +++ b/elasticsearch/templates/statefulset-master.yaml @@ -127,8 +127,12 @@ spec: key: ELASTICSEARCH_PASSWORD {{- end }} {{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} {{- end }} +{{- end }} {{- if .Values.pod.env.master }} {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index ba6bc08c2..864695725 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -978,4 +978,5 @@ manifests: service_logging: true statefulset_data: true statefulset_master: true + object_bucket_claim: false ... diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index c4f39edf9..e4b45e31c 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.55 +version: 0.2.56 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl new file mode 100644 index 000000000..08521e0fe --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }} +{{- range $s3Bucket := .Values.storage.s3.buckets }} +- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_ACCESS_KEY_ID +- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_SECRET_ACCESS_KEY +{{- end }} +{{- end }} diff --git a/releasenotes/notes/elasticsearch.yaml b/releasenotes/notes/elasticsearch.yaml index e7f48fd83..cc5b0a125 100644 --- a/releasenotes/notes/elasticsearch.yaml +++ b/releasenotes/notes/elasticsearch.yaml @@ -37,4 +37,5 @@ elasticsearch: - 0.2.25 Update ElasticSearch to 8.9.0 - 0.2.26 Add 2023.1 Ubuntu Focal overrides - 0.2.27 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.2.28 Utilize bucket claim CRD when using with Rook ... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index 2f002e03d..44e26149d 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -62,4 +62,5 @@ helm-toolkit: - 0.2.53 Update create db user queries - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice - 0.2.55 Updated deprecated IngressClass annotation + - 0.2.56 Expose S3 credentials from Rook bucket CRD secret ... diff --git a/roles/deploy-env/tasks/control-plane.yaml b/roles/deploy-env/tasks/control-plane.yaml index e9d7422ca..c722f92f6 100644 --- a/roles/deploy-env/tasks/control-plane.yaml +++ b/roles/deploy-env/tasks/control-plane.yaml @@ -19,7 +19,7 @@ shell: | mkdir -p /home/{{ kubectl.user }}/.kube cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config - chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config + chown -R {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube args: executable: /bin/bash diff --git a/tools/deployment/ceph/ceph-rook.sh b/tools/deployment/ceph/ceph-rook.sh index bae24d949..e519643e5 100755 --- a/tools/deployment/ceph/ceph-rook.sh +++ b/tools/deployment/ceph/ceph-rook.sh @@ -600,6 +600,8 @@ cephObjectStores: - name: default namespace: ceph spec: + allowUsersInNamespaces: + - "*" metadataPool: failureDomain: host replicated: @@ -693,18 +695,40 @@ bootstrap: manifests: daemonset_mon: false daemonset_osd: false - deployment_checkdns: true deployment_mds: false deployment_mgr: false deployment_mgr_sa: false deployment_moncheck: false helm_tests: false job_bootstrap: false - job_storage_admin_keys: true service_mgr: false service_mon: false service_mon_discovery: true + job_storage_admin_keys: true + job_keyring: true EOF helm upgrade --install ceph-mon ./ceph-mon --namespace=ceph --values=/tmp/ceph-supplemental.yaml ./tools/deployment/common/wait-for-pods.sh ceph + +# credentials for this object store user will be placed +# to the rook-ceph-object-user-default-s3-admin secret +# AccessKey is the secret field where the access key is stored +# SecretKey is the secret field where the secret key is stored +# cat > /tmp/s3_admin.yaml <