From 1d34fbba2a09dc8f9b3b0fed02259882f72a6ce0 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Thu, 20 Jun 2024 03:13:18 -0500 Subject: [PATCH] Test job for legacy OSH Ceph to Rook migration At the moment the recommended way of managing Ceph clusters is using Rook-Ceph operator. However some of the users still utilize legacy OSH Ceph* charts. Since Ceph is a critical part of the infrastructure we suggest a migration procedure and this PR is to test it. Change-Id: I837c8707b9fa45ff4350641920649188be1ce8da --- ceph-mon/Chart.yaml | 2 +- .../bin/keys/_storage-keyring-manager.sh.tpl | 1 + .../templates/job-storage-admin-keys.yaml | 2 + ceph-mon/values.yaml | 1 + ceph-provisioners/Chart.yaml | 2 +- .../templates/daemonset-csi-rbd-plugin.yaml | 2 +- .../deployment-csi-rbd-provisioner.yaml | 2 +- playbooks/inject-keys.yaml | 11 + releasenotes/notes/ceph-mon.yaml | 1 + releasenotes/notes/ceph-provisioners.yaml | 1 + tools/deployment/ceph/ceph-ns-activate.sh | 8 +- tools/deployment/ceph/ceph.sh | 1 - tools/deployment/ceph/ceph_legacy.sh | 198 ++++++ tools/deployment/ceph/migrate-after.sh | 29 + tools/deployment/ceph/migrate-before.sh | 34 + tools/deployment/ceph/migrate-to-rook-ceph.sh | 19 +- tools/deployment/ceph/migrate-values.sh | 621 ++++++++++++++++++ tools/deployment/common/rabbitmq.sh | 9 +- tools/deployment/db/mariadb.sh | 6 +- zuul.d/jobs.yaml | 46 ++ zuul.d/project.yaml | 1 + 21 files changed, 977 insertions(+), 20 deletions(-) create mode 100644 playbooks/inject-keys.yaml create mode 100755 tools/deployment/ceph/ceph_legacy.sh create mode 100755 tools/deployment/ceph/migrate-after.sh create mode 100755 tools/deployment/ceph/migrate-before.sh create mode 100755 tools/deployment/ceph/migrate-values.sh diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index fb7c7a28c..5a6eba2db 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.34 +version: 0.1.35 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl index b8cb6f506..431af1ab8 100644 --- a/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl +++ b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl @@ -91,6 +91,7 @@ EOF } #create_kube_storage_key create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME} +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME_NODE} {{ else }} diff --git a/ceph-mon/templates/job-storage-admin-keys.yaml b/ceph-mon/templates/job-storage-admin-keys.yaml index 2d782d434..0456f54e1 100644 --- a/ceph-mon/templates/job-storage-admin-keys.yaml +++ b/ceph-mon/templates/job-storage-admin-keys.yaml @@ -96,6 +96,8 @@ spec: value: {{ .Values.secrets.keyrings.admin }} - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME value: {{ .Values.storageclass.rbd.parameters.adminSecretName }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME_NODE + value: {{ .Values.storageclass.rbd.parameters.adminSecretNameNode }} command: - /tmp/keys-storage-keyring-manager.sh volumeMounts: diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index 225f43e06..ce523e844 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -436,6 +436,7 @@ storageclass: rbd: parameters: adminSecretName: pvc-ceph-conf-combined-storageclass + adminSecretNameNode: pvc-ceph-conf-combined-storageclass cephfs: provision_storage_class: true provisioner: ceph.com/cephfs diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml index 97c4b5b94..a95c70ba0 100644 --- a/ceph-provisioners/Chart.yaml +++ b/ceph-provisioners/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Provisioner name: ceph-provisioners -version: 0.1.29 +version: 0.1.30 home: https://github.com/ceph/ceph ... diff --git a/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml b/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml index 04557ebbc..1c92c348b 100644 --- a/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml +++ b/ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml @@ -109,7 +109,7 @@ spec: - "--nodeserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=0" - - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com" + - "--drivername={{ $envAll.Values.storageclass.csi_rbd.provisioner }}" - "--pidlimit=-1" env: - name: DEPLOYMENT_NAMESPACE diff --git a/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml index b5bff8ca8..d3de193f9 100644 --- a/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml @@ -231,7 +231,7 @@ spec: - "--controllerserver=true" - "--endpoint=$(CSI_ENDPOINT)" - "--v=0" - - "--drivername=$(DEPLOYMENT_NAMESPACE).rbd.csi.ceph.com" + - "--drivername={{ $envAll.Values.storageclass.csi_rbd.provisioner }}" - "--pidlimit=-1" env: - name: DEPLOYMENT_NAMESPACE diff --git a/playbooks/inject-keys.yaml b/playbooks/inject-keys.yaml new file mode 100644 index 000000000..c9a85b261 --- /dev/null +++ b/playbooks/inject-keys.yaml @@ -0,0 +1,11 @@ +--- +- hosts: all + tasks: + - name: Put keys to .ssh/authorized_keys + lineinfile: + path: /home/zuul/.ssh/authorized_keys + state: present + line: "{{ item }}" + loop: + - "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMyM6sgu/Xgg+VaLJX5c6gy6ynYX7pO7XNobnKotYRulcEkmiLprvLSg+WP25VDAcSoif3rek3qiVnEYh6R2/Go= vlad@russell" +... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 145b2eea1..e28a266fe 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -35,4 +35,5 @@ ceph-mon: - 0.1.32 Update Rook to 1.12.5 and Ceph to 18.2.0 - 0.1.33 Update Ceph images to Jammy and Reef 18.2.1 - 0.1.34 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.35 Use seprate secrets for CSI plugin and CSI provisioner ... diff --git a/releasenotes/notes/ceph-provisioners.yaml b/releasenotes/notes/ceph-provisioners.yaml index 015ec9c00..25f87c9b2 100644 --- a/releasenotes/notes/ceph-provisioners.yaml +++ b/releasenotes/notes/ceph-provisioners.yaml @@ -29,4 +29,5 @@ ceph-provisioners: - 0.1.27 Update Rook to 1.12.5 and Ceph to 18.2.0 - 0.1.28 Update Ceph images to Jammy and Reef 18.2.1 - 0.1.29 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.30 Specify CSI drivername in values.yaml ... diff --git a/tools/deployment/ceph/ceph-ns-activate.sh b/tools/deployment/ceph/ceph-ns-activate.sh index 77b4e7296..642723ea9 100755 --- a/tools/deployment/ceph/ceph-ns-activate.sh +++ b/tools/deployment/ceph/ceph-ns-activate.sh @@ -14,8 +14,10 @@ set -xe +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_PATH:="../openstack-helm-infra"} + #NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} tee /tmp/ceph-openstack-config.yaml < /tmp/ceph-fs-uuid.txt CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" #NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this diff --git a/tools/deployment/ceph/ceph_legacy.sh b/tools/deployment/ceph/ceph_legacy.sh new file mode 100755 index 000000000..3da0f0481 --- /dev/null +++ b/tools/deployment/ceph/ceph_legacy.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${CEPH_OSD_DATA_DEVICE:="/dev/loop100"} +: ${POD_NETWORK_CIDR:="10.244.0.0/24"} +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_PATH:="../openstack-helm-infra"} + +NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)" + +#NOTE: Deploy command +[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt +CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi + +# Most of PV fields are immutable and in case of CSI RBD plugin they refer +# to secrets which were used for RBD provisioner and RBD attacher. These fields +# can not be updated later. +# So for testing purposes we assume legacy Ceph cluster is deployed with +# the following secret names for the CSI plugin +# - rook-csi-rbd-provisioner +# - rook-csi-rbd-node +# These exact secret names are used by Rook by default for CSI plugin and +# and after migration PVs will be adopted by the new Rook Ceph cluster. +# +# Alternatively if we deploy legacy Ceph cluster with the default values +# then we could later force Rook to use same CSI secret names as used for +# legacy cluster. For example pvc-ceph-conf-combined-storageclass secret +# name is used by default in legacy charts. +# +# Same is for CSI provisioner drivername option. For testing we deploy +# legacy cluster with the drivername set to rook-ceph.rbd.csi.ceph.com +# while default value is ceph.rbd.csi.ceph.com. +# This is also for the sake of smooth adoption of PVs. + +tee /tmp/ceph.yaml <