openstack/openstack-helm-infra
Code Issues Proposed changes

Simplify ceph-adapter-rook

Browse Source 
- Do not deploy anything in the ceph namespace
- Prepare admin key secret in the openstack namespace.
  Get admin key from the Ceph tools pod
- Prepare Ceph client config with the mon_host
  taken from the rook-ceph-mon-endpoints configmap
  as recommended in the Rook documentation.

Change-Id: Idd4134efab49de032a389283e611c4959a6cbf24
This commit is contained in:
Vladimir Kozhukalov 2024-06-17 08:02:03 -05:00
parent b946e5ba05
commit b460c559bb
16 changed files with 95 additions and 496 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceph-adapter-rook/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Adapter Rook
name: ceph-adapter-rook
version: 0.1.2
version: 0.1.3
home: https://github.com/ceph/ceph
...

41
ceph-adapter-rook/README.md
View File

@ -7,45 +7,12 @@ via CRDs which can be used for managing pools/keys/users etc.
However Openstack-Helm charts do not utilize Rook CRDs but instead
manage Ceph assets like pools/keyrings/users/buckets etc. by means
of running bootstrap scripts. Before using Openstack-Helm charts we
have to provision a minimal set of assets like Ceph admin keys and
endpoints and this chart provides exactly this minimal set of templates.
have to provision a minimal set of assets like Ceph admin key and
Ceph client config.
# Usage
Deploy Ceph admin key and Ceph mon endpoint in the namespace where Ceph cluster is deployed.
```
tee > /tmp/ceph-adapter-rook-ceph.yaml <<EOF
manifests:
configmap_bin: true
configmap_templates: true
configmap_etc: false
job_storage_admin_keys: true
job_namespace_client_key: false
job_namespace_client_ceph_config: false
service_mon_discovery: true
EOF
helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
--namespace=ceph \
--values=/tmp/ceph-adapter-ceph.yaml
--namespace=openstack
```
Now wait until all jobs are finished and deploy client key and client
configuration in the namespace where Openstack charts are going to be deployed.
tee > /tmp/ceph-adapter-rook-openstack.yaml <<EOF
manifests:
configmap_bin: true
configmap_templates: false
configmap_etc: true
job_storage_admin_keys: false
job_namespace_client_key: true
job_namespace_client_ceph_config: true
service_mon_discovery: false
EOF
helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
--namespace=openstack \
--values=/tmp/ceph-adapter-rook-openstack.yaml
```
Again wait until all jobs are finished and then you can deploy other Openstack-Helm charts.
Once all the jobs are finished you can deploy other Openstack-Helm charts.

14
ceph-adapter-rook/templates/bin/_namespace-client-ceph-config-manager.sh.tpl → ceph-adapter-rook/templates/bin/_config-manager.sh.tpl
View File

@ -17,20 +17,10 @@ limitations under the License.
set -ex
{{- $envAll := . }}
{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }}
# TODO: Get endpoint from rook-ceph-mon-endpoints configmap
ENDPOINT=$(mon_host_from_k8s_ep ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} ceph-mon-discovery)
if [ -z "$ENDPOINT" ]; then
echo "Ceph Mon endpoint is empty"
exit 1
else
echo $ENDPOINT
fi
ENDPOINTS=$(kubectl --namespace ${CEPH_CLUSTER_NAMESPACE} get configmap rook-ceph-mon-endpoints -o jsonpath='{.data.data}' | sed 's/.=//g')
kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml | \
sed "s#mon_host.*#mon_host = ${ENDPOINT}#g" | \
sed "s#mon_host.*#mon_host = ${ENDPOINTS}#g" | \
kubectl apply -f -
kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml

17
ceph-adapter-rook/templates/bin/_namespace-client-key-manager.sh.tpl → ceph-adapter-rook/templates/bin/_key-manager.sh.tpl
View File

@ -17,16 +17,9 @@ limitations under the License.
set -ex
{{- $envAll := . }}
CEPH_RBD_KEY=$(kubectl get secret ${PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME} \
--namespace=${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} \
-o json )
# CONNECT_TO_ROOK_CEPH_CLUSTER is unset by default
if [[ ${CONNECT_TO_ROOK_CEPH_CLUSTER} == "true" ]] ; then
CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data["ceph-secret"]')
else
CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data.key')
fi
# We expect rook-ceph-tools pod to be up and running
ROOK_CEPH_TOOLS_POD=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}')
CEPH_ADMIN_KEY=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "client.admin" | awk '/key:/{print $2}')
ceph_activate_namespace() {
kube_namespace=$1
@ -43,9 +36,9 @@ metadata:
{{ tuple $envAll "ceph" "rbd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
type: "${secret_type}"
data:
key: $( echo ${ceph_key} )
key: $( echo ${ceph_key} | base64 | tr -d '\n' )
EOF
} | kubectl apply --namespace ${kube_namespace} -f -
}
ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "${CEPH_CLUSTER_KEY}"
ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${SECRET_NAME} "${CEPH_ADMIN_KEY}"

91
ceph-adapter-rook/templates/bin/_storage-keyring-manager.sh.tpl
View File

@ -1,91 +0,0 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
{{ if .Release.IsInstall }}
{{- $envAll := . }}
function kube_ceph_keyring_gen () {
CEPH_KEY=$1
CEPH_KEY_TEMPLATE=$2
sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n'
}
CEPH_CLIENT_KEY=""
ROOK_CEPH_TOOLS_POD=$(kubectl -n ${DEPLOYMENT_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}')
if [[ -n "${ROOK_CEPH_TOOLS_POD}" ]]; then
CEPH_AUTH_KEY_NAME=$(echo "${CEPH_KEYRING_NAME}" | awk -F. '{print $2 "." $3}')
CEPH_CLIENT_KEY=$(kubectl -n ${DEPLOYMENT_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "${CEPH_AUTH_KEY_NAME}" | awk '/key:/{print $2}')
fi
function create_kube_key () {
CEPH_KEYRING=$1
CEPH_KEYRING_NAME=$2
CEPH_KEYRING_TEMPLATE=$3
KUBE_SECRET_NAME=$4
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
{
cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
name: ${KUBE_SECRET_NAME}
labels:
{{ tuple $envAll "ceph" "admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
type: Opaque
data:
${CEPH_KEYRING_NAME}: $( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
EOF
} | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f -
fi
}
#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME}
function create_kube_storage_key () {
CEPH_KEYRING=$1
KUBE_SECRET_NAME=$2
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
{
cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
name: ${KUBE_SECRET_NAME}
labels:
{{ tuple $envAll "ceph" "admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
type: kubernetes.io/rbd
data:
key: $( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
userID: $( echo -n "admin" | base64 | tr -d '\n' )
userKey: $( echo -n ${CEPH_KEYRING} | base64 | tr -d '\n' )
EOF
} | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f -
fi
}
#create_kube_storage_key <ceph_key> <kube_secret_name>
create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME}
{{ else }}
echo "Not touching ${KUBE_SECRET_NAME} as this is not the initial deployment"
{{ end }}

10
ceph-adapter-rook/templates/configmap-bin.yaml
View File

@ -20,11 +20,9 @@ kind: ConfigMap
metadata:
name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
data:
keys-storage-keyring-manager.sh: |
{{ tuple "bin/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
provisioner-rbd-namespace-client-key-manager.sh: |
{{ tuple "bin/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
provisioner-rbd-namespace-client-ceph-config-manager.sh: |
{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
key-manager.sh: |
{{ tuple "bin/_key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
config-manager.sh: |
{{ tuple "bin/_config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

4
ceph-adapter-rook/templates/configmap-etc-client.yaml
View File

@ -44,6 +44,6 @@ data:
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- list .Values.ceph_configmap_name . | include "ceph.configmap.etc" }}
{{- if .Values.manifests.configmap_etc_client }}
{{- list .Values.configmap_name . | include "ceph.configmap.etc" }}
{{- end }}

25
ceph-adapter-rook/templates/configmap-templates.yaml
View File

@ -1,25 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_templates }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }}
data:
admin.keyring: |
{{ .Values.conf.templates.keyring.admin | indent 4 }}
{{- end }}

40
ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml
View File

@ -17,8 +17,8 @@ limitations under the License.
{{- $randStringSuffix := randAlphaNum 5 | lower }}
{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-ceph-config-generator" }}
{{ tuple $envAll "namespace_client_ceph_config_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $serviceAccountName := print $envAll.Release.Name "-namespace-client-ceph-config" }}
{{ tuple $envAll "namespace_client_ceph_config" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
@ -52,12 +52,12 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.admin_secret_namespace }}
namespace: {{ .Values.ceph_cluster_namespace }}
rules:
- apiGroups:
- ""
resources:
- endpoints
- configmaps
verbs:
- get
- list
@ -66,7 +66,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.admin_secret_namespace }}
namespace: {{ .Values.ceph_cluster_namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -81,53 +81,53 @@ kind: Job
metadata:
name: {{ $serviceAccountName }}
labels:
{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ tuple $envAll "ceph" "namespace-client-ceph-config" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
{{ tuple $envAll "ceph" "namespace-client-ceph-config" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "client_ceph_config_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
{{ dict "envAll" $envAll "application" "namespace_client_ceph_config" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "namespace_client_ceph_config_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{ tuple $envAll "namespace-client-ceph-config-init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: ceph-storage-keys-generator
- name: namespace-client-ceph-config
{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "client_ceph_config_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.namespace_client_ceph_config | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "namespace_client_ceph_config" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: CEPH_CONF_ETC
value: {{ .Values.ceph_configmap_name }}
value: {{ .Values.configmap_name }}
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.admin_secret_namespace }}
- name: CEPH_CLUSTER_NAMESPACE
value: {{ .Values.ceph_cluster_namespace }}
command:
- /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh
- /tmp/config-manager.sh
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-provisioners-bin-clients
mountPath: /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh
subPath: provisioner-rbd-namespace-client-ceph-config-manager.sh
- name: bin
mountPath: /tmp/config-manager.sh
subPath: config-manager.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: pod-etc-ceph
emptyDir: {}
- name: ceph-provisioners-bin-clients
- name: bin
configMap:
name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
defaultMode: 0555

50
ceph-adapter-rook/templates/job-namespace-client-key.yaml
View File

@ -17,8 +17,8 @@ limitations under the License.
{{- $randStringSuffix := randAlphaNum 5 | lower }}
{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-key-generator" }}
{{ tuple $envAll "namespace_client_key_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $serviceAccountName := print $envAll.Release.Name "-namespace-client-key" }}
{{ tuple $envAll "namespace-client-key" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
@ -52,21 +52,27 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.admin_secret_namespace }}
namespace: {{ .Values.ceph_cluster_namespace }}
rules:
- apiGroups:
- ""
resources:
- secrets
- pods
verbs:
- get
- list
- apiGroups:
- ""
resources:
- pods/exec
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.admin_secret_namespace }}
namespace: {{ .Values.ceph_cluster_namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -81,55 +87,53 @@ kind: Job
metadata:
name: {{ $serviceAccountName }}
labels:
{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ tuple $envAll "ceph" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
{{ tuple $envAll "ceph" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "client_key_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
{{ dict "envAll" $envAll "application" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "namespace_client_key_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{ tuple $envAll "namespace-client-key-init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: ceph-storage-keys-generator
- name: namespace-client-key
{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "client_key_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.namespace_client_key | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "namespace-client-key" "container" "namespace-client-key" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.secrets.keys.user }}
- name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.secrets.keys.admin }}
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.admin_secret_namespace }}
- name: SECRET_NAME
value: {{ .Values.secret_name }}
- name: CEPH_CLUSTER_NAMESPACE
value: {{ .Values.ceph_cluster_namespace }}
command:
- /tmp/provisioner-rbd-namespace-client-key-manager.sh
- /tmp/key-manager.sh
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-provisioners-bin-clients
mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh
subPath: provisioner-rbd-namespace-client-key-manager.sh
- name: bin
mountPath: /tmp/key-manager.sh
subPath: key-manager.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: pod-etc-ceph
emptyDir: {}
- name: ceph-provisioners-bin-clients
- name: bin
configMap:
name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
defaultMode: 0555

128
ceph-adapter-rook/templates/job-storage-admin-keys.yaml
View File

@ -1,128 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_storage_admin_keys }}
{{- $envAll := . }}
{{- $serviceAccountName := "ceph-storage-keys-generator" }}
{{ tuple $envAll "storage_keys_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ $serviceAccountName }}
namespace: {{ .Values.admin_secret_namespace }}
rules:
- apiGroups:
- ""
resources:
- pods
- pods/exec
- secrets
verbs:
- get
- create
- patch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ $serviceAccountName }}
namespace: {{ .Values.admin_secret_namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: ceph-storage-keys-generator
namespace: {{ .Values.admin_secret_namespace }}
labels:
{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
{{ dict "envAll" $envAll "podName" "ceph-storage-keys-generator" "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "storage_keys_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: ceph-storage-keys-generator
{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "storage_keys_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CEPH_GEN_DIR
value: /tmp
- name: CEPH_TEMPLATES_DIR
value: /tmp/templates
- name: CEPH_KEYRING_NAME
value: ceph.client.admin.keyring
- name: CEPH_KEYRING_TEMPLATE
value: admin.keyring
- name: CEPH_KEYRING_ADMIN_NAME
value: {{ .Values.secrets.keyrings.admin }}
- name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.secrets.keys.admin }}
command:
- /tmp/keys-storage-keyring-manager.sh
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: pod-etc-ceph
mountPath: /etc/ceph
- name: ceph-mon-bin
mountPath: /tmp/keys-storage-keyring-manager.sh
subPath: keys-storage-keyring-manager.sh
readOnly: true
- name: ceph-templates
mountPath: /tmp/templates
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: pod-etc-ceph
emptyDir: {}
- name: ceph-mon-bin
configMap:
name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }}
defaultMode: 0555
- name: ceph-templates
configMap:
name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }}
defaultMode: 0444
{{- end }}

37
ceph-adapter-rook/templates/service-mon-discovery.yaml
View File

@ -1,37 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.service_mon_discovery }}
{{- $envAll := . }}
---
kind: Service
apiVersion: v1
metadata:
name: {{ tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
- name: mon
port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
protocol: TCP
targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- name: mon-msgr2
port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
protocol: TCP
targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
app: rook-ceph-mon
ceph_daemon_type: mon
clusterIP: None
publishNotReadyAddresses: true
{{- end }}

86
ceph-adapter-rook/values.yaml
View File

@ -18,33 +18,25 @@ labels:
pod:
security_context:
storage_keys_generator:
pod:
runAsUser: 65534
container:
ceph_storage_keys_generator:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
client_key_generator:
namespace_client_key:
pod:
runAsUser: 99
container:
ceph_storage_keys_generator:
namespace_client_key:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
dns_policy: "ClusterFirstWithHostNet"
affinity:
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
weight:
default: 10
resources:
enabled: false
jobs:
secret_provisioning:
namespace_client_key:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "128Mi"
cpu: "500m"
namespace_client_ceph_config:
limits:
memory: "1024Mi"
cpu: "2000m"
@ -52,68 +44,28 @@ pod:
memory: "128Mi"
cpu: "500m"
secrets:
keyrings:
admin: ceph-client-admin-keyring
keys:
admin: pvc-ceph-conf-combined-storageclass
user: pvc-ceph-client-key
admin_secret_namespace: ceph
ceph_configmap_name: ceph-etc
ceph_cluster_namespace: ceph
secret_name: pvc-ceph-client-key
configmap_name: ceph-etc
conf:
templates:
keyring:
admin: |
[client.admin]
key = {{ key }}
auid = 0
caps mds = "allow"
caps mon = "allow *"
caps osd = "allow *"
caps mgr = "allow *"
ceph:
global:
# auth
cephx: true
cephx_require_signatures: false
cephx_cluster_require_signatures: true
cephx_service_require_signatures: false
objecter_inflight_op_bytes: "1073741824"
objecter_inflight_ops: 10240
debug_ms: "0/0"
log_file: /dev/stdout
mon_cluster_log_file: /dev/stdout
# TODO: Get mon host from rook-ceph-mon-endpoints configmap
mon_host: "will be discovered"
endpoints:
cluster_domain_suffix: cluster.local
ceph_mon:
namespace: ceph
hosts:
default: ceph-mon
discovery: ceph-mon-discovery
host_fqdn_override:
default: null
port:
mon:
default: 6789
mon_msgr2:
default: 3300
dependencies:
static:
storage_keys_generator:
namespace_client_key:
jobs: null
namespace_client_ceph_config:
jobs: null
manifests:
configmap_bin: true
configmap_templates: true
configmap_etc: true
job_storage_admin_keys: true
job_namespace_client_key: true
configmap_etc_client: true
job_namespace_client_ceph_config: true
service_mon_discovery: true
job_namespace_client_key: true
...

1
releasenotes/notes/ceph-adapter-rook.yaml
View File

@ -3,4 +3,5 @@ ceph-adapter-rook:
- 0.1.0 Initial Chart
- 0.1.1 Update Ceph images to Jammy and Reef 18.2.1
- 0.1.2 Update Ceph images to patched 18.2.2 and restore debian-reef repo
- 0.1.3 Simplify and remove unnecessary entities
...

38
tools/deployment/ceph/ceph-adapter-rook.sh
View File

@ -15,40 +15,12 @@
set -xe
make ceph-adapter-rook
#NOTE: Define variables
: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"}
: ${OSH_INFRA_PATH:="../openstack-helm-infra"}
tee > /tmp/ceph-adapter-rook-ceph.yaml <<EOF
manifests:
configmap_bin: true
configmap_templates: true
configmap_etc: false
job_storage_admin_keys: true
job_namespace_client_key: false
job_namespace_client_ceph_config: false
service_mon_discovery: true
EOF
helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
--namespace=ceph \
--values=/tmp/ceph-adapter-rook-ceph.yaml
#NOTE: Wait for deploy
helm osh wait-for-pods ceph
tee > /tmp/ceph-adapter-rook-openstack.yaml <<EOF
manifests:
configmap_bin: true
configmap_templates: false
configmap_etc: true
job_storage_admin_keys: false
job_namespace_client_key: true
job_namespace_client_ceph_config: true
service_mon_discovery: false
EOF
helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \
--namespace=openstack \
--values=/tmp/ceph-adapter-rook-openstack.yaml
helm upgrade --install ceph-adapter-rook ${OSH_INFRA_HELM_REPO}/ceph-adapter-rook \
--namespace=openstack
#NOTE: Wait for deploy
helm osh wait-for-pods openstack

7
zuul.d/jobs.yaml
View File

@ -273,6 +273,8 @@
- ^helm-toolkit/.*
- ^roles/.*
- ^ceph.*
- ^tools/deployment/ceph/ceph\.sh$
- ^tools/deployment/ceph/ceph-ns-activate\.sh$
- job:
name: openstack-helm-infra-cinder-2024-1-ubuntu_jammy
@ -283,8 +285,9 @@
files:
- ^helm-toolkit/.*
- ^roles/.*
- ^tools/deployment/ceph-rook\.sh$
- ^tools/deployment/ceph-adapter-rook\.sh$
- ^ceph.*
- ^tools/deployment/ceph/ceph-rook\.sh$
- ^tools/deployment/ceph/ceph-adapter-rook\.sh$
- job:
name: openstack-helm-infra-tls-2024-1-ubuntu_jammy