Simplify ceph-adapter-rook

- Do not deploy anything in the ceph namespace
- Prepare admin key secret in the openstack namespace.
  Get admin key from the Ceph tools pod
- Prepare Ceph client config with the mon_host
  taken from the rook-ceph-mon-endpoints configmap
  as recommended in the Rook documentation.

Change-Id: Idd4134efab49de032a389283e611c4959a6cbf24

ceph-adapter-rook/Chart.yaml

@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph Adapter Rook
 name: ceph-adapter-rook
-version: 0.1.2
+version: 0.1.3
 home: https://github.com/ceph/ceph
 ...

ceph-adapter-rook/README.md

@@ -7,45 +7,12 @@ via CRDs which can be used for managing pools/keys/users etc.
 However Openstack-Helm charts do not utilize Rook CRDs but instead
 manage Ceph assets like pools/keyrings/users/buckets etc. by means
 of running bootstrap scripts. Before using Openstack-Helm charts we
-have to provision a minimal set of assets like Ceph admin keys and
+have to provision a minimal set of assets like Ceph admin key and
-endpoints and this chart provides exactly this minimal set of templates.
+Ceph client config.
 
 # Usage
-Deploy Ceph admin key and Ceph mon endpoint in the namespace where Ceph cluster is deployed.
-```
-tee > /tmp/ceph-adapter-rook-ceph.yaml <<EOF
-manifests:
-  configmap_bin: true
-  configmap_templates: true
-  configmap_etc: false
-  job_storage_admin_keys: true
-  job_namespace_client_key: false
-  job_namespace_client_ceph_config: false
-  service_mon_discovery: true
-EOF
-
 helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
-  --namespace=ceph \
+  --namespace=openstack
-  --values=/tmp/ceph-adapter-ceph.yaml
 ```
 
-Now wait until all jobs are finished and deploy client key and client
+Once all the jobs are finished you can deploy other Openstack-Helm charts.
-configuration in the namespace where Openstack charts are going to be deployed.
-
-tee > /tmp/ceph-adapter-rook-openstack.yaml <<EOF
-manifests:
-  configmap_bin: true
-  configmap_templates: false
-  configmap_etc: true
-  job_storage_admin_keys: false
-  job_namespace_client_key: true
-  job_namespace_client_ceph_config: true
-  service_mon_discovery: false
-EOF
-
-helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
-  --namespace=openstack \
-  --values=/tmp/ceph-adapter-rook-openstack.yaml
-```
-
-Again wait until all jobs are finished and then you can deploy other Openstack-Helm charts.

ceph-adapter-rook/templates/bin/_config-manager.sh.tpl

@@ -17,20 +17,10 @@ limitations under the License.
 set -ex
 {{- $envAll := . }}
 
-{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
+ENDPOINTS=$(kubectl --namespace ${CEPH_CLUSTER_NAMESPACE} get configmap rook-ceph-mon-endpoints -o jsonpath='{.data.data}' | sed 's/.=//g')
-
-# TODO: Get endpoint from rook-ceph-mon-endpoints configmap
-ENDPOINT=$(mon_host_from_k8s_ep ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} ceph-mon-discovery)
-
-if [ -z "$ENDPOINT" ]; then
-  echo "Ceph Mon endpoint is empty"
-  exit 1
-else
-  echo $ENDPOINT
-fi
 
 kubectl get cm ${CEPH_CONF_ETC} -n  ${DEPLOYMENT_NAMESPACE}  -o yaml | \
-  sed "s#mon_host.*#mon_host = ${ENDPOINT}#g" | \
+  sed "s#mon_host.*#mon_host = ${ENDPOINTS}#g" | \
   kubectl apply -f -
 
 kubectl get cm ${CEPH_CONF_ETC} -n  ${DEPLOYMENT_NAMESPACE}  -o yaml

ceph-adapter-rook/templates/bin/_key-manager.sh.tpl

@@ -17,16 +17,9 @@ limitations under the License.
 set -ex
 {{- $envAll := . }}
 
-CEPH_RBD_KEY=$(kubectl get secret ${PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME} \
+# We expect rook-ceph-tools pod to be up and running
-    --namespace=${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} \
+ROOK_CEPH_TOOLS_POD=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}')
-    -o json )
+CEPH_ADMIN_KEY=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "client.admin" | awk '/key:/{print $2}')
-
-# CONNECT_TO_ROOK_CEPH_CLUSTER is unset by default
-if [[ ${CONNECT_TO_ROOK_CEPH_CLUSTER} == "true" ]] ; then
-  CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data["ceph-secret"]')
-else
-  CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data.key')
-fi
 
 ceph_activate_namespace() {
   kube_namespace=$1
@@ -43,9 +36,9 @@ metadata:
 {{ tuple $envAll "ceph" "rbd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 type: "${secret_type}"
 data:
-  key: $( echo ${ceph_key} )
+  key: $( echo ${ceph_key} | base64 | tr -d '\n' )
 EOF
   } | kubectl apply --namespace ${kube_namespace} -f -
 }
 
-ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "${CEPH_CLUSTER_KEY}"
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${SECRET_NAME} "${CEPH_ADMIN_KEY}"

ceph-adapter-rook/templates/bin/_storage-keyring-manager.sh.tpl

@@ -1,91 +0,0 @@
-#!/bin/bash
-
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-set -ex
-{{ if .Release.IsInstall }}
-{{- $envAll := . }}
-
-function kube_ceph_keyring_gen () {
-  CEPH_KEY=$1
-  CEPH_KEY_TEMPLATE=$2
-  sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n'
-}
-
-CEPH_CLIENT_KEY=""
-ROOK_CEPH_TOOLS_POD=$(kubectl -n ${DEPLOYMENT_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}')
-
-if [[ -n "${ROOK_CEPH_TOOLS_POD}" ]]; then
-  CEPH_AUTH_KEY_NAME=$(echo "${CEPH_KEYRING_NAME}" | awk -F. '{print $2 "." $3}')
-  CEPH_CLIENT_KEY=$(kubectl -n ${DEPLOYMENT_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "${CEPH_AUTH_KEY_NAME}" | awk '/key:/{print $2}')
-fi
-
-function create_kube_key () {
-  CEPH_KEYRING=$1
-  CEPH_KEYRING_NAME=$2
-  CEPH_KEYRING_TEMPLATE=$3
-  KUBE_SECRET_NAME=$4
-
-  if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
-    {
-      cat <<EOF
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: ${KUBE_SECRET_NAME}
-  labels:
-{{ tuple $envAll "ceph" "admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-type: Opaque
-data:
-  ${CEPH_KEYRING_NAME}: $( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
-EOF
-    } | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f -
-  fi
-}
-#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
-create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME}
-
-function create_kube_storage_key () {
-  CEPH_KEYRING=$1
-  KUBE_SECRET_NAME=$2
-
-  if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
-    {
-      cat <<EOF
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: ${KUBE_SECRET_NAME}
-  labels:
-{{ tuple $envAll "ceph" "admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-type: kubernetes.io/rbd
-data:
-  key: $( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
-  userID: $( echo -n "admin" | base64 | tr -d '\n' )
-  userKey: $( echo -n ${CEPH_KEYRING} | base64 | tr -d '\n' )
-EOF
-    } | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f -
-  fi
-}
-#create_kube_storage_key <ceph_key> <kube_secret_name>
-create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME}
-
-{{ else }}
-
-echo "Not touching ${KUBE_SECRET_NAME} as this is not the initial deployment"
-
-{{ end }}

ceph-adapter-rook/templates/configmap-bin.yaml

@@ -20,11 +20,9 @@ kind: ConfigMap
 metadata:
   name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
 data:
-  keys-storage-keyring-manager.sh: |
+  key-manager.sh: |
-{{ tuple "bin/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ tuple "bin/_key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  provisioner-rbd-namespace-client-key-manager.sh: |
+  config-manager.sh: |
-{{ tuple "bin/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ tuple "bin/_config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  provisioner-rbd-namespace-client-ceph-config-manager.sh: |
-{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
 {{- end }}

ceph-adapter-rook/templates/configmap-etc-client.yaml

@@ -44,6 +44,6 @@ data:
 {{- end }}
 {{- end }}
 
-{{- if .Values.manifests.configmap_etc }}
+{{- if .Values.manifests.configmap_etc_client }}
-{{- list  .Values.ceph_configmap_name . | include "ceph.configmap.etc" }}
+{{- list  .Values.configmap_name . | include "ceph.configmap.etc" }}
 {{- end }}

ceph-adapter-rook/templates/configmap-templates.yaml

@@ -1,25 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.configmap_templates }}
-{{- $envAll := . }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }}
-data:
-  admin.keyring: |
-{{ .Values.conf.templates.keyring.admin | indent 4 }}
-{{- end }}

ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml

@@ -17,8 +17,8 @@ limitations under the License.
 
 {{- $randStringSuffix := randAlphaNum 5 | lower }}
 
-{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-ceph-config-generator" }}
+{{- $serviceAccountName := print $envAll.Release.Name "-namespace-client-ceph-config" }}
-{{ tuple $envAll "namespace_client_ceph_config_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{ tuple $envAll "namespace_client_ceph_config" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -52,12 +52,12 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.admin_secret_namespace }}
+  namespace: {{ .Values.ceph_cluster_namespace }}
 rules:
   - apiGroups:
       - ""
     resources:
-      - endpoints
+      - configmaps
     verbs:
       - get
       - list
@@ -66,7 +66,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.admin_secret_namespace }}
+  namespace: {{ .Values.ceph_cluster_namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -81,53 +81,53 @@ kind: Job
 metadata:
   name: {{ $serviceAccountName }}
   labels:
-{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "ceph" "namespace-client-ceph-config" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "ceph" "namespace-client-ceph-config" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
 {{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
-{{ dict "envAll" $envAll "application" "client_ceph_config_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "namespace_client_ceph_config" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
       nodeSelector:
         {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
       initContainers:
-{{ tuple $envAll "namespace_client_ceph_config_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{ tuple $envAll "namespace-client-ceph-config-init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
-        - name: ceph-storage-keys-generator
+        - name: namespace-client-ceph-config
 {{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.namespace_client_ceph_config | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "client_ceph_config_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "namespace_client_ceph_config" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           env:
             - name: CEPH_CONF_ETC
-              value: {{ .Values.ceph_configmap_name }}
+              value: {{ .Values.configmap_name }}
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
+            - name: CEPH_CLUSTER_NAMESPACE
-              value: {{ .Values.admin_secret_namespace }}
+              value: {{ .Values.ceph_cluster_namespace }}
           command:
-            - /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh
+            - /tmp/config-manager.sh
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-provisioners-bin-clients
+            - name: bin
-              mountPath: /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh
+              mountPath: /tmp/config-manager.sh
-              subPath: provisioner-rbd-namespace-client-ceph-config-manager.sh
+              subPath: config-manager.sh
               readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
         - name: pod-etc-ceph
           emptyDir: {}
-        - name: ceph-provisioners-bin-clients
+        - name: bin
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
             defaultMode: 0555

ceph-adapter-rook/templates/job-namespace-client-key.yaml

@@ -17,8 +17,8 @@ limitations under the License.
 
 {{- $randStringSuffix := randAlphaNum 5 | lower }}
 
-{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-key-generator" }}
+{{- $serviceAccountName := print $envAll.Release.Name "-namespace-client-key" }}
-{{ tuple $envAll "namespace_client_key_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+{{ tuple $envAll "namespace-client-key" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -52,21 +52,27 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.admin_secret_namespace }}
+  namespace: {{ .Values.ceph_cluster_namespace }}
 rules:
   - apiGroups:
       - ""
     resources:
-      - secrets
+      - pods
     verbs:
       - get
       - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.admin_secret_namespace }}
+  namespace: {{ .Values.ceph_cluster_namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -81,55 +87,53 @@ kind: Job
 metadata:
   name: {{ $serviceAccountName }}
   labels:
-{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ tuple $envAll "ceph" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
   template:
     metadata:
       labels:
-{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+{{ tuple $envAll "ceph" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
 {{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
-{{ dict "envAll" $envAll "application" "client_key_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+{{ dict "envAll" $envAll "application" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $serviceAccountName }}
       restartPolicy: OnFailure
       nodeSelector:
         {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
       initContainers:
-{{ tuple $envAll "namespace_client_key_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{ tuple $envAll "namespace-client-key-init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
-        - name: ceph-storage-keys-generator
+        - name: namespace-client-key
 {{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.namespace_client_key | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "client_key_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "application" "namespace-client-key" "container" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           env:
             - name: DEPLOYMENT_NAMESPACE
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
+            - name: SECRET_NAME
-              value: {{ .Values.secrets.keys.user }}
+              value: {{ .Values.secret_name }}
-            - name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
+            - name: CEPH_CLUSTER_NAMESPACE
-              value: {{ .Values.secrets.keys.admin }}
+              value: {{ .Values.ceph_cluster_namespace }}
-            - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
-              value: {{ .Values.admin_secret_namespace }}
           command:
-            - /tmp/provisioner-rbd-namespace-client-key-manager.sh
+            - /tmp/key-manager.sh
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-provisioners-bin-clients
+            - name: bin
-              mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh
+              mountPath: /tmp/key-manager.sh
-              subPath: provisioner-rbd-namespace-client-key-manager.sh
+              subPath: key-manager.sh
               readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
         - name: pod-etc-ceph
           emptyDir: {}
-        - name: ceph-provisioners-bin-clients
+        - name: bin
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
             defaultMode: 0555

ceph-adapter-rook/templates/job-storage-admin-keys.yaml

@@ -1,128 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if .Values.manifests.job_storage_admin_keys }}
-{{- $envAll := . }}
-
-{{- $serviceAccountName := "ceph-storage-keys-generator" }}
-{{ tuple $envAll "storage_keys_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ $serviceAccountName }}
-  namespace: {{ .Values.admin_secret_namespace }}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/exec
-      - secrets
-    verbs:
-      - get
-      - create
-      - patch
-      - list
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ $serviceAccountName }}
-  namespace: {{ .Values.admin_secret_namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: {{ $serviceAccountName }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ $serviceAccountName }}
-    namespace: {{ $envAll.Release.Namespace }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: ceph-storage-keys-generator
-  namespace: {{ .Values.admin_secret_namespace }}
-  labels:
-{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
-  annotations:
-    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-spec:
-  template:
-    metadata:
-      labels:
-{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
-      annotations:
-{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
-        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
-{{ dict "envAll" $envAll "podName" "ceph-storage-keys-generator" "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
-    spec:
-{{ dict "envAll" $envAll "application" "storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
-      serviceAccountName: {{ $serviceAccountName }}
-      restartPolicy: OnFailure
-      nodeSelector:
-        {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
-      initContainers:
-{{ tuple $envAll "storage_keys_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-      containers:
-        - name: ceph-storage-keys-generator
-{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
-{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-{{ dict "envAll" $envAll "application" "storage_keys_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          env:
-            - name: DEPLOYMENT_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CEPH_GEN_DIR
-              value: /tmp
-            - name: CEPH_TEMPLATES_DIR
-              value: /tmp/templates
-            - name: CEPH_KEYRING_NAME
-              value: ceph.client.admin.keyring
-            - name: CEPH_KEYRING_TEMPLATE
-              value: admin.keyring
-            - name: CEPH_KEYRING_ADMIN_NAME
-              value: {{ .Values.secrets.keyrings.admin }}
-            - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
-              value: {{ .Values.secrets.keys.admin }}
-          command:
-            - /tmp/keys-storage-keyring-manager.sh
-          volumeMounts:
-            - name: pod-tmp
-              mountPath: /tmp
-            - name: pod-etc-ceph
-              mountPath: /etc/ceph
-            - name: ceph-mon-bin
-              mountPath: /tmp/keys-storage-keyring-manager.sh
-              subPath: keys-storage-keyring-manager.sh
-              readOnly: true
-            - name: ceph-templates
-              mountPath: /tmp/templates
-              readOnly: true
-      volumes:
-        - name: pod-tmp
-          emptyDir: {}
-        - name: pod-etc-ceph
-          emptyDir: {}
-        - name: ceph-mon-bin
-          configMap:
-            name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
-            defaultMode: 0555
-        - name: ceph-templates
-          configMap:
-            name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }}
-            defaultMode: 0444
-{{- end }}

ceph-adapter-rook/templates/service-mon-discovery.yaml

@@ -1,37 +0,0 @@
-{{/*
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- if and .Values.manifests.service_mon_discovery }}
-{{- $envAll := . }}
----
-kind: Service
-apiVersion: v1
-metadata:
-  name: {{ tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-spec:
-  ports:
-  - name: mon
-    port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-    protocol: TCP
-    targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-  - name: mon-msgr2
-    port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-    protocol: TCP
-    targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-  selector:
-    app: rook-ceph-mon
-    ceph_daemon_type: mon
-  clusterIP: None
-  publishNotReadyAddresses: true
-{{- end }}

ceph-adapter-rook/values.yaml

@@ -18,33 +18,25 @@ labels:
 
 pod:
   security_context:
-    storage_keys_generator:
+    namespace_client_key:
-      pod:
-        runAsUser: 65534
-      container:
-        ceph_storage_keys_generator:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-    client_key_generator:
       pod:
         runAsUser: 99
       container:
-        ceph_storage_keys_generator:
+        namespace_client_key:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
   dns_policy: "ClusterFirstWithHostNet"
-  affinity:
-    anti:
-      type:
-        default: preferredDuringSchedulingIgnoredDuringExecution
-      topologyKey:
-        default: kubernetes.io/hostname
-      weight:
-        default: 10
   resources:
     enabled: false
     jobs:
-      secret_provisioning:
+      namespace_client_key:
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+        requests:
+          memory: "128Mi"
+          cpu: "500m"
+      namespace_client_ceph_config:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
@@ -52,68 +44,28 @@ pod:
           memory: "128Mi"
           cpu: "500m"
 
-secrets:
-  keyrings:
-    admin: ceph-client-admin-keyring
-  keys:
-    admin: pvc-ceph-conf-combined-storageclass
-    user: pvc-ceph-client-key
 
-admin_secret_namespace: ceph
+ceph_cluster_namespace: ceph
-ceph_configmap_name: ceph-etc
+
+secret_name: pvc-ceph-client-key
+configmap_name: ceph-etc
 
 conf:
-  templates:
-    keyring:
-      admin: |
-        [client.admin]
-          key = {{ key }}
-          auid = 0
-          caps mds = "allow"
-          caps mon = "allow *"
-          caps osd = "allow *"
-          caps mgr = "allow *"
   ceph:
     global:
-      # auth
-      cephx: true
-      cephx_require_signatures: false
-      cephx_cluster_require_signatures: true
-      cephx_service_require_signatures: false
-      objecter_inflight_op_bytes: "1073741824"
-      objecter_inflight_ops: 10240
-      debug_ms: "0/0"
-      log_file: /dev/stdout
-      mon_cluster_log_file: /dev/stdout
       # TODO: Get mon host from rook-ceph-mon-endpoints configmap
       mon_host: "will be discovered"
 
-endpoints:
-  cluster_domain_suffix: cluster.local
-  ceph_mon:
-    namespace: ceph
-    hosts:
-      default: ceph-mon
-      discovery: ceph-mon-discovery
-    host_fqdn_override:
-      default: null
-    port:
-      mon:
-        default: 6789
-      mon_msgr2:
-        default: 3300
-
 dependencies:
   static:
-    storage_keys_generator:
+    namespace_client_key:
+      jobs: null
+    namespace_client_ceph_config:
       jobs: null
 
 manifests:
   configmap_bin: true
-  configmap_templates: true
+  configmap_etc_client: true
-  configmap_etc: true
-  job_storage_admin_keys: true
-  job_namespace_client_key: true
   job_namespace_client_ceph_config: true
-  service_mon_discovery: true
+  job_namespace_client_key: true
 ...

releasenotes/notes/ceph-adapter-rook.yaml

@@ -3,4 +3,5 @@ ceph-adapter-rook:
   - 0.1.0 Initial Chart
   - 0.1.1 Update Ceph images to Jammy and Reef 18.2.1
   - 0.1.2 Update Ceph images to patched 18.2.2 and restore debian-reef repo
+  - 0.1.3 Simplify and remove unnecessary entities
 ...

tools/deployment/ceph/ceph-adapter-rook.sh

@@ -15,40 +15,12 @@
 
 set -xe
 
-make ceph-adapter-rook
+#NOTE: Define variables
+: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"}
+: ${OSH_INFRA_PATH:="../openstack-helm-infra"}
 
-tee > /tmp/ceph-adapter-rook-ceph.yaml <<EOF
+helm upgrade --install ceph-adapter-rook ${OSH_INFRA_HELM_REPO}/ceph-adapter-rook \
-manifests:
+  --namespace=openstack
-  configmap_bin: true
-  configmap_templates: true
-  configmap_etc: false
-  job_storage_admin_keys: true
-  job_namespace_client_key: false
-  job_namespace_client_ceph_config: false
-  service_mon_discovery: true
-EOF
-
-helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
-  --namespace=ceph \
-  --values=/tmp/ceph-adapter-rook-ceph.yaml
-
-#NOTE: Wait for deploy
-helm osh wait-for-pods ceph
-
-tee > /tmp/ceph-adapter-rook-openstack.yaml <<EOF
-manifests:
-  configmap_bin: true
-  configmap_templates: false
-  configmap_etc: true
-  job_storage_admin_keys: false
-  job_namespace_client_key: true
-  job_namespace_client_ceph_config: true
-  service_mon_discovery: false
-EOF
-
-helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
-  --namespace=openstack \
-  --values=/tmp/ceph-adapter-rook-openstack.yaml
 
 #NOTE: Wait for deploy
 helm osh wait-for-pods openstack

zuul.d/jobs.yaml

@@ -273,6 +273,8 @@
       - ^helm-toolkit/.*
       - ^roles/.*
       - ^ceph.*
+      - ^tools/deployment/ceph/ceph\.sh$
+      - ^tools/deployment/ceph/ceph-ns-activate\.sh$
 
 - job:
     name: openstack-helm-infra-cinder-2024-1-ubuntu_jammy
@@ -283,8 +285,9 @@
     files:
       - ^helm-toolkit/.*
       - ^roles/.*
-      - ^tools/deployment/ceph-rook\.sh$
+      - ^ceph.*
-      - ^tools/deployment/ceph-adapter-rook\.sh$
+      - ^tools/deployment/ceph/ceph-rook\.sh$
+      - ^tools/deployment/ceph/ceph-adapter-rook\.sh$
 
 - job:
     name: openstack-helm-infra-tls-2024-1-ubuntu_jammy