diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index 2ed9b165e..44aa39c38 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.4 +version: 0.1.5 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl index 598033253..dfa85f437 100644 --- a/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl +++ b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl @@ -73,6 +73,8 @@ metadata: type: kubernetes.io/rbd data: key: $( echo ${CEPH_KEYRING} | base64 | tr -d '\n' ) + userID: $( echo -n "admin" | base64 | tr -d '\n' ) + userKey: $( echo -n ${CEPH_KEYRING} | base64 | tr -d '\n' ) EOF } | kubectl apply --namespace ${DEPLOYMENT_NAMESPACE} -f - fi diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml index ab7fe7bd3..b714ea5b8 100644 --- a/ceph-provisioners/Chart.yaml +++ b/ceph-provisioners/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Provisioner name: ceph-provisioners -version: 0.1.3 +version: 0.1.4 home: https://github.com/ceph/ceph ... diff --git a/ceph-provisioners/templates/bin/_helm-tests.sh.tpl b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl index 72510f31a..0c8c2be57 100644 --- a/ceph-provisioners/templates/bin/_helm-tests.sh.tpl +++ b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl @@ -172,6 +172,7 @@ EOF reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME +reset_test_env $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME reset_test_env $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME {{- range $storageclass, $val := .Values.storageclass }} @@ -184,6 +185,14 @@ then reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME fi +if [ {{ $val.provisioner }} == "ceph.rbd.csi.ceph.com" ] && [ {{ $val.provision_storage_class }} == true ]; +then + echo "--> Checking CSI RBD storage class." + storageclass={{ $val.metadata.name }} + storageclass_validation $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME $storageclass + reset_test_env $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME +fi + if [ {{ $val.provisioner }} == "ceph.com/cephfs" ] && [ {{ $val.provision_storage_class }} == true ]; then echo "--> Checking cephfs storage class." diff --git a/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl index 8fa24d0ba..421e6f61a 100644 --- a/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl +++ b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl @@ -46,5 +46,5 @@ if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${PVC_CEPH_CEPHFS_S ${DEPLOYMENT_NAMESPACE} \ "kubernetes.io/cephfs" \ ${PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME} \ - "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data | .[]')" + "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data.key')" fi diff --git a/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl index 1846f51fb..e6a8abeab 100644 --- a/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl +++ b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl @@ -41,4 +41,4 @@ EOF } | kubectl apply --namespace ${kube_namespace} -f - } -ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data | .[]')" +ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data.key')" diff --git a/ceph-provisioners/templates/configmap-etc-csi.yaml b/ceph-provisioners/templates/configmap-etc-csi.yaml new file mode 100644 index 000000000..a37800d82 --- /dev/null +++ b/ceph-provisioners/templates/configmap-etc-csi.yaml @@ -0,0 +1,48 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc.csi" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if and (.Values.deployment.ceph) (.Values.deployment.csi) }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +data: + config.json: |- + [ + { + "clusterID": {{ .Release.Namespace | quote }}, + "monitors": [ + {{ .Values.conf.ceph.global.mon_host | quote }} + ] + } + ] +metadata: + name: ceph-csi-config +{{- end }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_etc }} +{{- list .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }} +{{- end }} diff --git a/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml b/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml new file mode 100644 index 000000000..295903239 --- /dev/null +++ b/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml @@ -0,0 +1,181 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-nodeplugin" }} +{{ tuple $envAll "rbd_provisioner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ceph-rbd-plugin + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "plugin" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "rbd" "plugin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rbd-plugin" "containerNames" (list "driver-registrar" "csi-rbdplugin" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "plugin" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.csi_rbd_plugin.node_selector_key }}: {{ .Values.labels.csi_rbd_plugin.node_selector_value }} + hostNetwork: true + hostPID: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "rbd_plugin" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: driver-registrar +{{ tuple $envAll "csi_registrar" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_registrar | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "plugin" "container" "ceph_rbd_registrar" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--v=0" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com/csi.sock" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-rbdplugin +{{ tuple $envAll "cephcsi" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_cephcsi | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "plugin" "container" "ceph_csi_rbd_plugin" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--nodeserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=0" + - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com" + - "--pidlimit=-1" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /run/mount + name: host-mount + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/ceph.rbd.csi.ceph.com + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins + type: Directory + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: host-mount + hostPath: + path: /run/mount + - name: lib-modules + hostPath: + path: /lib/modules + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- end }} diff --git a/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml new file mode 100644 index 000000000..2f120aca8 --- /dev/null +++ b/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml @@ -0,0 +1,283 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-provisioner" }} +{{ tuple $envAll "rbd_provisioner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }}-run-rbd-provisioner +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ $envAll.Release.Namespace }} + name: {{ $serviceAccountName }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: Role + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-rbd-csi-provisioner + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.rbd_provisioner }} + selector: + matchLabels: +{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rbd-csi-provisioner" "containerNames" (list "ceph-rbd-provisioner" "ceph-rbd-snapshotter" "ceph-rbd-attacher" "csi-resizer" "csi-rbdplugin" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.provisioner.node_selector_key }}: {{ .Values.labels.provisioner.node_selector_value }} + initContainers: +{{ tuple $envAll "rbd_provisioner" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-rbd-provisioner +{{ tuple $envAll "csi_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + + args: + - "--csi-address=$(ADDRESS)" + - "--v=0" + - "--timeout=150s" + - "--retry-interval-start=500ms" + - "--enable-leader-election=true" + - "--leader-election-type=leases" + - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: ceph-rbd-snapshotter +{{ tuple $envAll "csi_snapshotter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_snapshotter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_snapshotter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=0" + - "--timeout=150s" + - "--leader-election=true" + - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: ceph-rbd-attacher +{{ tuple $envAll "csi_attacher" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_attacher | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_attacher" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--v=0" + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--retry-interval-start=500ms" + - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ADDRESS + value: /csi/csi-provisioner.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer +{{ tuple $envAll "csi_resizer" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_resizer | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_resizer" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=0" + - "--csiTimeout=150s" + - "--leader-election" + - "--leader-election-namespace=$(DEPLOYMENT_NAMESPACE)" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ADDRESS + value: unix:///csi/csi-provisioner.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-rbdplugin +{{ tuple $envAll "cephcsi" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rbd_cephcsi | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_cephcsi" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--nodeid=$(NODE_ID)" + - "--type=rbd" + - "--controllerserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=0" + - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com" + - "--pidlimit=-1" + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi-provisioner.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ + - name: keys-tmp-dir + mountPath: /tmp/csi/keys + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + - name: socket-dir + emptyDir: { + medium: "Memory" + } + - name: ceph-csi-config + configMap: + name: ceph-csi-config + - name: keys-tmp-dir + emptyDir: { + medium: "Memory" + } +{{- end }} diff --git a/ceph-provisioners/templates/pod-helm-tests.yaml b/ceph-provisioners/templates/pod-helm-tests.yaml index 8141b282d..3edb521be 100644 --- a/ceph-provisioners/templates/pod-helm-tests.yaml +++ b/ceph-provisioners/templates/pod-helm-tests.yaml @@ -88,6 +88,10 @@ spec: value: {{ .Values.pod.test_pod.rbd.name }} - name: RBD_TEST_PVC_NAME value: {{ .Values.pod.test_pod.rbd.pvc_name }} + - name: CSI_RBD_TEST_POD_NAME + value: {{ .Values.pod.test_pod.csi_rbd.name }} + - name: CSI_RBD_TEST_PVC_NAME + value: {{ .Values.pod.test_pod.csi_rbd.pvc_name }} - name: CEPHFS_TEST_POD_NAME value: {{ .Values.pod.test_pod.cephfs.name }} - name: CEPHFS_TEST_PVC_NAME diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml index 6fc372747..f9241b325 100644 --- a/ceph-provisioners/values.yaml +++ b/ceph-provisioners/values.yaml @@ -20,6 +20,7 @@ deployment: ceph: true client_secrets: false rbd_provisioner: true + csi: true cephfs_provisioner: true release_group: null @@ -31,6 +32,12 @@ images: ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_bionic-20200521' ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113' ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:change_770201_ubuntu_bionic-20210113' + csi_provisioner: 'quay.io/k8scsi/csi-provisioner:v1.6.0' + csi_snapshotter: 'quay.io/k8scsi/csi-snapshotter:v2.1.1' + csi_attacher: 'quay.io/k8scsi/csi-attacher:v2.1.1' + csi_resizer: 'quay.io/k8scsi/csi-resizer:v0.4.0' + csi_registrar: 'quay.io/k8scsi/csi-node-driver-registrar:v1.2.0' + cephcsi: 'quay.io/cephcsi/cephcsi:v3.1.0' dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0' image_repo_sync: 'docker.io/docker:17.07.0' local_registry: @@ -49,6 +56,9 @@ labels: provisioner: node_selector_key: openstack-control-plane node_selector_value: enabled + csi_rbd_plugin: + node_selector_key: openstack-control-plane + node_selector_value: enabled pod: test_pod: @@ -56,6 +66,9 @@ pod: rbd: name: rbd-prov-test-pod pvc_name: rbd-prov-test-pvc + csi_rbd: + name: csi-rbd-prov-test-pod + pvc_name: csi-rbd-prov-test-pvc cephfs: name: cephfs-prov-test-pod pvc_name: cephfs-prov-test-pvc @@ -70,6 +83,29 @@ pod: ceph_rbd_provisioner: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + ceph_rbd_snapshotter: + privileged: true + ceph_rbd_attacher: + privileged: true + ceph_rbd_resizer: + privileged: true + ceph_rbd_cephcsi: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + plugin: + pod: + runAsUser: 0 + container: + ceph_rbd_registrar: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + ceph_csi_rbd_plugin: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true bootstrap: pod: runAsUser: 99 @@ -112,6 +148,12 @@ pod: upgrades: deployments: pod_replacement_strategy: Recreate + daemonsets: + pod_replacement_strategy: RollingUpdate + plugin: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 affinity: anti: type: @@ -136,6 +178,41 @@ pod: limits: memory: "50Mi" cpu: "500m" + rbd_attacher: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rbd_registrar: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rbd_resizer: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rbd_snapshotter: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rbd_cephcsi: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" jobs: bootstrap: limits: @@ -263,8 +340,7 @@ storageclass: provisioner: ceph.com/rbd ceph_configmap_name: ceph-etc metadata: - default_storage_class: true - name: general + name: general-rbd parameters: pool: rbd adminId: admin @@ -274,6 +350,24 @@ storageclass: userSecretName: pvc-ceph-client-key imageFormat: "2" imageFeatures: layering + csi_rbd: + provision_storage_class: true + provisioner: ceph.rbd.csi.ceph.com + metadata: + default_storage_class: true + name: general + parameters: + clusterID: ceph + csi.storage.k8s.io/controller-expand-secret-name: pvc-ceph-conf-combined-storageclass + csi.storage.k8s.io/controller-expand-secret-namespace: ceph + csi.storage.k8s.io/fstype: ext4 + csi.storage.k8s.io/node-stage-secret-name: pvc-ceph-conf-combined-storageclass + csi.storage.k8s.io/node-stage-secret-namespace: ceph + csi.storage.k8s.io/provisioner-secret-name: pvc-ceph-conf-combined-storageclass + csi.storage.k8s.io/provisioner-secret-namespace: ceph + imageFeatures: layering + imageFormat: "2" + pool: rbd cephfs: provision_storage_class: true provisioner: ceph.com/cephfs @@ -317,6 +411,7 @@ manifests: configmap_bin_common: true configmap_etc: true deployment_rbd_provisioner: true + deployment_csi_rbd_provisioner: true deployment_cephfs_provisioner: true job_bootstrap: false job_cephfs_client_key: true diff --git a/ceph-provisioners/values_overrides/apparmor.yaml b/ceph-provisioners/values_overrides/apparmor.yaml index e4e101506..0d3ed7256 100644 --- a/ceph-provisioners/values_overrides/apparmor.yaml +++ b/ceph-provisioners/values_overrides/apparmor.yaml @@ -11,6 +11,13 @@ pod: ceph-rbd-provisioner: ceph-rbd-provisioner: runtime/default init: runtime/default + ceph-rbd-csi-provisioner: + ceph-rbd-provisioner: runtime/default + init: runtime/default + ceph-rbd-snapshotter: runtime/default + ceph-rbd-attacher: runtime/default + csi-resizer: runtime/default + csi-rbdplugin: runtime/default ceph-provisioner-test: init: runtime/default ceph-provisioner-helm-test: runtime/default @@ -20,6 +27,10 @@ pod: ceph-provisioners-ceph-ns-key-generator: ceph-storage-keys-generator: runtime/default init: runtime/default + ceph-rbd-plugin: + driver-registrar: runtime/default + csi-rbdplugin: runtime/default + init: runtime/default deployment: client_secrets: true diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 33febfa58..1e46e4e6a 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -5,4 +5,5 @@ ceph-mon: - 0.1.2 Enable shareProcessNamespace in mon daemonset - 0.1.3 Run mon container as ceph user - 0.1.4 Uplift from Nautilus to Octopus release + - 0.1.5 Add Ceph CSI plugin ... diff --git a/releasenotes/notes/ceph-provisioners.yaml b/releasenotes/notes/ceph-provisioners.yaml index 14ce22aa9..30f9762cb 100644 --- a/releasenotes/notes/ceph-provisioners.yaml +++ b/releasenotes/notes/ceph-provisioners.yaml @@ -4,4 +4,5 @@ ceph-provisioners: - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" - 0.1.2 Validate each storageclass created - 0.1.3 Uplift from Nautilus to Octopus release + - 0.1.4 Add Ceph CSI plugin ...