starlingx/platform-armada-app
Code Issues Proposed changes

Merge "Add SnapshotClass Creation for CephFS/RBD via Helm Override"

Browse Source
This commit is contained in:
Zuul 2024-01-08 16:42:44 +00:00 committed by Gerrit Code Review
commit 396dfb7d34
12 changed files with 248 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
platform-helm/debian/deb_folder/patches/0003-ceph-csi-cephfs-add-storage-init.yaml.patch
View File

@ -1,11 +1,12 @@
From 239cfdde94db341aad0ccb15e420c4ecc3157d21 Mon Sep 17 00:00:00 2001
From: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Date: Mon, 24 Apr 2023 16:57:46 -0300
From 356ddba07d33e333d332415d9858f4c6354a0d85 Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Wed, 6 Dec 2023 14:16:19 -0300
Subject: [PATCH] ceph-csi-cephfs: add storage-init.yaml
Signed-off-by: Hediberto Cavalcante da Silva <hediberto.cavalcantedasilva@windriver.com>
Signed-off-by: Felipe Sanches Zanoni <Felipe.SanchesZanoni@windriver.com>
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../templates/storage-init.yaml | 369 ++++++++++++++++++
1 file changed, 369 insertions(+)
@ -344,7 +345,7 @@ index 000000000..97052d7a9
+ name: cephfs-storage-init
+ defaultMode: 0555
+ containers:
+ {{- range $sc := .Values.classes }}
+ {{- range $sc := .Values.storageClasses }}
+ - name: storage-init-{{- $sc.name }}
+ image: "{{ $.Values.storage_init.image.repository }}:{{ $.Values.storage_init.image.tag }}"
+ command: ["/bin/bash", "/tmp/storage-init.sh"]

9
platform-helm/debian/deb_folder/patches/0008-ceph-csi-rbd-add-storage-init.yaml.patch
View File

@ -1,11 +1,12 @@
From 4e3d4d76c16c4d03ba33afd0581ceac5f5ea3147 Mon Sep 17 00:00:00 2001
From: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Date: Mon, 24 Apr 2023 17:04:02 -0300
From 080cad7da551b36ee22139c558770cf6520a0d55 Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Wed, 6 Dec 2023 14:17:19 -0300
Subject: [PATCH] ceph-csi-rbd: add storage-init.yaml
Signed-off-by: Hediberto Cavalcante da Silva <hediberto.cavalcantedasilva@windriver.com>
Signed-off-by: Felipe Sanches Zanoni <Felipe.SanchesZanoni@windriver.com>
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../ceph-csi-rbd/templates/storage-init.yaml | 394 ++++++++++++++++++
1 file changed, 394 insertions(+)
@ -372,7 +373,7 @@ index 000000000..6abd8ec3f
+ configMap:
+ name: rbd-storage-init
+ containers:
+ {{- range $sc := .Values.classes }}
+ {{- range $sc := .Values.storageClasses }}
+ - name: storage-init-{{- $sc.name }}
+ image: "{{ $.Values.storage_init.image.repository }}:{{ $.Values.storage_init.image.tag }}"
+ command: [ "/bin/bash", "/tmp/storage-init.sh" ]

9
platform-helm/debian/deb_folder/patches/0011-ceph-csi-rbd-update-storageclass.patch
View File

@ -1,10 +1,11 @@
From e02f28d37a57550e4e4127c5b3758e865cfc4d7e Mon Sep 17 00:00:00 2001
From: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Date: Mon, 20 Mar 2023 17:56:25 -0300
From d4ecc92cf187225ab4cdb882370647cb5b5d0d3b Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Wed, 6 Dec 2023 14:22:19 -0300
Subject: [PATCH] ceph-csi-rbd: update storageclass
Signed-off-by: Hediberto Cavalcante da Silva <hediberto.cavalcantedasilva@windriver.com>
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../ceph-csi-rbd/templates/storageclass.yaml | 162 ++++++++++--------
1 file changed, 93 insertions(+), 69 deletions(-)
@ -17,7 +18,7 @@ index a559456aa..b53c3dc6c 100644
{{- if .Values.storageClass.create -}}
+{{- $cephCsiDefault := .Values.storageClass }}
+{{- $defaults := .Values.classdefaults }}
+{{- range $sc := .Values.classes }}
+{{- range $sc := .Values.storageClasses }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:

9
platform-helm/debian/deb_folder/patches/0012-ceph-csi-cephfs-update-storageclass.patch
View File

@ -1,10 +1,11 @@
From 8ef1e051d4166dd0f9f8c931465b27cadfde7ca0 Mon Sep 17 00:00:00 2001
From: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Date: Tue, 21 Mar 2023 21:09:29 -0300
From 93ae57852cf8583228e99465c3d35e5304b88e65 Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Wed, 6 Dec 2023 14:24:22 -0300
Subject: [PATCH] ceph-csi-cephfs: update storageclass
Signed-off-by: Hediberto Cavalcante da Silva <hediberto.cavalcantedasilva@windriver.com>
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../templates/storageclass.yaml | 104 ++++++++++--------
1 file changed, 56 insertions(+), 48 deletions(-)
@ -17,7 +18,7 @@ index a21c99824..e3a27b455 100644
{{- if .Values.storageClass.create -}}
+{{- $cephCsiDefault := .Values.storageClass }}
+{{- $defaults := .Values.classdefaults }}
+{{- range $sc := .Values.classes }}
+{{- range $sc := .Values.storageClasses }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:

62
platform-helm/debian/deb_folder/patches/0015-ceph-csi-cephfs-add-snapshotclass.patch Normal file
View File

@ -0,0 +1,62 @@
From cc40abf2495bdaa0ace32441cfc6d5a78a397d3b Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Thu, 30 Nov 2023 12:51:37 -0300
Subject: ceph-csi-cephfs: add snapshotclass
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../templates/snapshotclass.yaml | 13 +++++++++++++
charts/ceph-csi-cephfs/values.yaml | 18 ++++++++++++++++++
2 files changed, 31 insertions(+)
create mode 100644 charts/ceph-csi-cephfs/templates/snapshotclass.yaml
diff --git a/charts/ceph-csi-cephfs/templates/snapshotclass.yaml b/charts/ceph-csi-cephfs/templates/snapshotclass.yaml
new file mode 100644
index 000000000..3bad19026
--- /dev/null
+++ b/charts/ceph-csi-cephfs/templates/snapshotclass.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.snapshotClass.create -}}
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+ name: {{ .Values.snapshotClass.name }}
+driver: {{ .Values.driverName }}
+parameters:
+ clusterID: {{ .Values.snapshotClass.clusterID }}
+ snapshotNamePrefix: {{ .Values.snapshotClass.snapshotNamePrefix }}
+ csi.storage.k8s.io/snapshotter-secret-name: {{ .Values.snapshotClass.provisionerSecret }}
+ csi.storage.k8s.io/snapshotter-secret-namespace: {{ .Values.snapshotClass.provisionerSecretNamespace }}
+deletionPolicy: {{ .Values.snapshotClass.deletionPolicy }}
+{{- end -}}
diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml
index 7c85abd7d..912165f1d 100644
--- a/charts/ceph-csi-cephfs/values.yaml
+++ b/charts/ceph-csi-cephfs/values.yaml
@@ -302,6 +302,24 @@ storageClass:
# Ceph pools name
metadata_pool: kube-cephfs-metadata
+snapshotClass:
+ # Specifies whether the snapshotclass should be created
+ create: false
+ # Specifies the snapshotclass name
+ name: csi-cephfsplugin-snapclass
+ # (required) Unique string representing a Ceph cluster to provision
+ #storage from.
+ clusterID: <cluster-ID>
+ # Prefix to use for naming CephFS snapshots.
+ # If omitted, defaults to "csi-snap-".
+ # snapshotNamePrefix: "foo-bar-"
+ snapshotNamePrefix: ""
+ # K8s secret name
+ provisionerSecret: csi-cephfs-secret
+ # K8s secret namespace
+ provisionerSecretNamespace: ""
+ deletionPolicy: Delete
+
secret:
# Specifies whether the secret should be created
create: false
--
2.25.1

62
platform-helm/debian/deb_folder/patches/0016-ceph-csi-rbd-add-snapshotclass.patch Normal file
View File

@ -0,0 +1,62 @@
From d6dbc163f3be00789a74f85556de4486792f9d72 Mon Sep 17 00:00:00 2001
From: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
Date: Thu, 30 Nov 2023 13:38:09 -0300
Subject: ceph-csi-rbd: add snapshotclass
Signed-off-by: Gabriel de Araújo Cabral <gabriel.cabral@windriver.com>
---
.../ceph-csi-rbd/templates/snapshotclass.yaml | 13 +++++++++++++
charts/ceph-csi-rbd/values.yaml | 18 ++++++++++++++++++
2 files changed, 31 insertions(+)
create mode 100644 charts/ceph-csi-rbd/templates/snapshotclass.yaml
diff --git a/charts/ceph-csi-rbd/templates/snapshotclass.yaml b/charts/ceph-csi-rbd/templates/snapshotclass.yaml
new file mode 100644
index 000000000..3bad19026
--- /dev/null
+++ b/charts/ceph-csi-rbd/templates/snapshotclass.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.snapshotClass.create -}}
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+ name: {{ .Values.snapshotClass.name }}
+driver: {{ .Values.driverName }}
+parameters:
+ clusterID: {{ .Values.snapshotClass.clusterID }}
+ snapshotNamePrefix: {{ .Values.snapshotClass.snapshotNamePrefix }}
+ csi.storage.k8s.io/snapshotter-secret-name: {{ .Values.snapshotClass.provisionerSecret }}
+ csi.storage.k8s.io/snapshotter-secret-namespace: {{ .Values.snapshotClass.provisionerSecretNamespace }}
+deletionPolicy: {{ .Values.snapshotClass.deletionPolicy }}
+{{- end -}}
diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml
index c33d5f2a6..8dd9ea3d5 100644
--- a/charts/ceph-csi-rbd/values.yaml
+++ b/charts/ceph-csi-rbd/values.yaml
@@ -449,6 +449,24 @@ storageClass:
# selinux-enabled filesystems
selinuxMount: true
+snapshotClass:
+ # Specifies whether the snapshotclass should be created
+ create: false
+ # Specifies the snapshotclass name
+ name: csi-rbdplugin-snapclass
+ # (required) Unique string representing a Ceph cluster to provision
+ #storage from.
+ clusterID: <cluster-ID>
+ # Prefix to use for naming RBD snapshots.
+ # If omitted, defaults to "csi-snap-".
+ # snapshotNamePrefix: "foo-bar-"
+ snapshotNamePrefix: ""
+ # K8s secret name
+ provisionerSecret: csi-rbd-secret
+ # K8s secret namespace
+ provisionerSecretNamespace: ""
+ deletionPolicy: Delete
+
secret:
# Specifies whether the secret should be created
create: false
--
2.25.1

2
platform-helm/debian/deb_folder/patches/series
View File

@ -12,3 +12,5 @@
0012-ceph-csi-cephfs-update-storageclass.patch
0013-ceph-csi-rbd-add-annotations-to-nodeplugin-daemonset.patch
0014-ceph-csi-cephfs-add-annotations-to-nodeplugin-daemonset.patch
0015-ceph-csi-cephfs-add-snapshotclass.patch
0016-ceph-csi-rbd-add-snapshotclass.patch

10
python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/ceph_fs_provisioner.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2020-2022 Wind River Systems, Inc.
# Copyright (c) 2020-2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -192,6 +192,11 @@ class CephFSProvisionerHelm(base.FluxCDBaseHelm):
storage_classes.append(storage_class)
snapshot_class = {
"clusterID": cluster_id,
"provisionerSecret": user_secret_name or class_defaults["adminSecretName"]
}
provisioner = {
"replicaCount": self._num_replicas_for_platform_app()
}
@ -206,7 +211,8 @@ class CephFSProvisionerHelm(base.FluxCDBaseHelm):
overrides = {
app_constants.HELM_NS_CEPH_FS_PROVISIONER: {
"classes": storage_classes,
"storageClasses": storage_classes,
"snapshotClass": snapshot_class,
"provisioner": provisioner,
"csiConfig": csi_config,
"classdefaults": class_defaults

10
python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/rbd_provisioner.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2020-2022 Wind River Systems, Inc.
# Copyright (c) 2020-2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -107,6 +107,11 @@ class RbdProvisionerHelm(base.FluxCDBaseHelm):
storage_classes.append(storage_class)
snapshot_class = {
"clusterID": cluster_id,
"provisionerSecret": user_secret_name or class_defaults["adminSecretName"]
}
provisioner = {
"replicaCount": self._num_replicas_for_platform_app()
}
@ -118,7 +123,8 @@ class RbdProvisionerHelm(base.FluxCDBaseHelm):
overrides = {
common.HELM_NS_RBD_PROVISIONER: {
"classes": storage_classes,
"storageClasses": storage_classes,
"snapshotClass": snapshot_class,
"provisioner": provisioner,
"csiConfig": csi_config,
"classdefaults": class_defaults

75
python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/lifecycle/lifecycle_platform.py
View File

@ -18,8 +18,10 @@ from sysinv.common import constants
from sysinv.common import exception
from sysinv.common import kubernetes
from sysinv.common import utils as cutils
from sysinv.db import api as dbapi
from sysinv.helm import lifecycle_base as base
from sysinv.helm import lifecycle_utils as lifecycle_utils
from k8sapp_platform.common import constants as app_constants
LOG = logging.getLogger(__name__)
@ -57,7 +59,7 @@ class PlatformAppLifecycleOperator(base.AppLifecycleOperator):
elif hook_info.lifecycle_type == constants.APP_LIFECYCLE_TYPE_RESOURCE:
if hook_info.operation == constants.APP_APPLY_OP and \
hook_info.relative_timing == constants.APP_LIFECYCLE_TIMING_PRE:
return lifecycle_utils.create_local_registry_secrets(app_op, app, hook_info)
return self.pre_apply(app_op, app, hook_info)
elif hook_info.operation == constants.APP_REMOVE_OP and \
hook_info.relative_timing == constants.APP_LIFECYCLE_TIMING_POST:
return lifecycle_utils.delete_local_registry_secrets(app_op, app, hook_info)
@ -99,6 +101,61 @@ class PlatformAppLifecycleOperator(base.AppLifecycleOperator):
raise exception.LifecycleSemanticCheckException(
"Not enough hosts in desired state")
def pre_apply(self, app_op, app, hook_info):
"""Pre Apply actions
Creates the local registry secret and rename user overrides from
'classes' to 'storageClasses' in the rbd and cephfs charts if
necessary.
:param app_op: AppOperator object
:param app: AppOperator.Application object
:param hook_info: LifecycleHookInfo object
"""
lifecycle_utils.create_local_registry_secrets(app_op, app, hook_info)
# TODO: The code below is for stx.8.0 -> stx.9.0 updates.
# It may be removed in the stx.10.0 release cycle
dbapi_instance = dbapi.get_instance()
# get most recently created inactive app
inactive_db_apps = dbapi_instance.kube_app_get_inactive(
app.name, limit=1, sort_key='created_at', sort_dir='desc')
if not inactive_db_apps:
# user overrides will not be updated because there is no
# inactive platform-integ-apps entry in the database
return
from_db_app = inactive_db_apps[0]
to_db_app = dbapi_instance.kube_app_get(app.name)
charts = [
app_constants.FLUXCD_HELMRELEASE_CEPH_FS_PROVISIONER,
app_constants.FLUXCD_HELMRELEASE_RBD_PROVISIONER,
]
# update of user overrides due to changing 'classes' to 'storageClasses'
# to improve understanding. The namespace of both charts are the same
for chart in charts:
user_overrides = self._get_helm_user_overrides(
dbapi_instance,
from_db_app,
chart,
app_constants.K8S_CEPHFS_PROVISIONER_DEFAULT_NAMESPACE)
if 'classes:' in user_overrides:
user_overrides = user_overrides.replace("classes:", "storageClasses:")
self._update_helm_user_overrides(
dbapi_instance,
to_db_app,
chart,
app_constants.K8S_CEPHFS_PROVISIONER_DEFAULT_NAMESPACE,
user_overrides,
)
LOG.debug("User overrides of 'classes' updated to 'storageClasses'"
" in {} chart from {}".format(chart, app.name))
def delete_csi_drivers(self, app):
""" Delete CSI drivers
@ -113,3 +170,19 @@ class PlatformAppLifecycleOperator(base.AppLifecycleOperator):
cmd = ["kubectl", "--kubeconfig", kubernetes.KUBERNETES_ADMIN_CONF, "delete", "csidriver", driver]
stdout, stderr = cutils.trycmd(*cmd)
LOG.debug("{} app: cmd={} stdout={} stderr={}".format(app.name, cmd, stdout, stderr))
def _get_helm_user_overrides(self, dbapi_instance, app, chart, namespace):
try:
return dbapi_instance.helm_override_get(
app_id=app.id,
name=chart,
namespace=namespace,
).user_overrides or ""
except exception.HelmOverrideNotFound:
# Override for this chart not found, nothing to be done
return ""
def _update_helm_user_overrides(self, dbapi_instance, app, chart, namespace, overrides):
values = {'user_overrides': overrides}
dbapi_instance.helm_override_update(
app_id=app.id, name=chart, namespace=namespace, values=values)

6
stx-platform-helm/stx-platform-helm/fluxcd-manifests/cephfs-provisioner/cephfs-provisioner-static-overrides.yaml
View File

@ -83,6 +83,12 @@ storageClass:
"helm.sh/hook-delete-policy": "before-hook-creation"
}
snapshotClass:
create: false
name: cephfs-snapshot
snapshotNamePrefix: cephfs-snap-
provisionerSecretNamespace: kube-system
topology:
enabled: false

6
stx-platform-helm/stx-platform-helm/fluxcd-manifests/rbd-provisioner/rbd-provisioner-static-overrides.yaml
View File

@ -84,6 +84,12 @@ storageClass:
mountOptions:
- discard
snapshotClass:
create: false
name: rbd-snapshot
snapshotNamePrefix: rbd-snap-
provisionerSecretNamespace: kube-system
topology:
enabled: false