From 3bde9f5b907b92a85a7188a1f11bf34820e6af52 Mon Sep 17 00:00:00 2001 From: Chinasubbareddy Mallavarapu Date: Wed, 13 May 2020 12:32:52 -0500 Subject: [PATCH] [CEPH] OSH-INFRA: use loopback devices for ceph osds - This is to make use of loopback devices for ceph osds since support for directory backed osds going to deprecate. - Move to bluestore from filestore for ceph-osds. - Seperate DB and WAL partitions from data so that gates will validate the scenario where we will have fast storage disk for DB and WAL. Change-Id: Ief6de17c53d6cb57ef604895fdc66dc6c604fd89 --- .../019-setup-ceph-loopback-device.sh | 1 + .../common/019-setup-ceph-loopback-device.sh | 13 ++++++++++++ .../019-setup-ceph-loopback-device.sh | 1 + .../019-setup-ceph-loopback-device.sh | 13 ++++++++++++ tools/deployment/multinode/030-ceph.sh | 16 +++++++++----- .../019-setup-ceph-loopback-device.sh | 1 + .../019-setup-ceph-loopback-device.sh | 1 + .../019-setup-ceph-loopback-device.sh | 1 + .../deployment/osh-infra-logging/020-ceph.sh | 16 +++++++++----- .../019-setup-ceph-loopback-device.sh | 21 +++++++++++++++++++ tools/deployment/tenant-ceph/030-ceph.sh | 15 ++++++++----- .../deployment/tenant-ceph/040-tenant-ceph.sh | 15 ++++++++----- zuul.d/jobs.yaml | 9 ++++++++ 13 files changed, 103 insertions(+), 20 deletions(-) create mode 120000 tools/deployment/apparmor/019-setup-ceph-loopback-device.sh create mode 100755 tools/deployment/common/019-setup-ceph-loopback-device.sh create mode 120000 tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh create mode 100755 tools/deployment/multinode/019-setup-ceph-loopback-device.sh create mode 120000 tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh create mode 120000 tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh create mode 120000 tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh create mode 100755 tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh diff --git a/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh b/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh new file mode 120000 index 000000000..3d8509fcc --- /dev/null +++ b/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh @@ -0,0 +1 @@ +../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/common/019-setup-ceph-loopback-device.sh b/tools/deployment/common/019-setup-ceph-loopback-device.sh new file mode 100755 index 000000000..d40cf5329 --- /dev/null +++ b/tools/deployment/common/019-setup-ceph-loopback-device.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -xe +sudo df -lh +sudo lsblk +sudo mkdir -p /var/lib/openstack-helm/ceph +sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +#lets check the devices +sudo df -lh +sudo lsblk diff --git a/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh b/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh new file mode 120000 index 000000000..436c5d6e2 --- /dev/null +++ b/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh @@ -0,0 +1 @@ +../multinode/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/multinode/019-setup-ceph-loopback-device.sh b/tools/deployment/multinode/019-setup-ceph-loopback-device.sh new file mode 100755 index 000000000..250ac8363 --- /dev/null +++ b/tools/deployment/multinode/019-setup-ceph-loopback-device.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -xe +sudo df -lh +sudo lsblk +sudo mkdir -p /var/lib/openstack-helm/ceph +sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +# lets check the devices +sudo df -lh +sudo lsblk diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index 57648cb4c..d844e4551 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -69,11 +69,15 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" + jobs: ceph_defragosds: # Execute every 15 minutes for gates @@ -94,6 +98,8 @@ manifests: cronjob_defragosds: true deployment_cephfs_provisioner: false job_cephfs_client_key: false +deploy: + tool: "ceph-volume" EOF for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do diff --git a/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh b/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh new file mode 120000 index 000000000..3d8509fcc --- /dev/null +++ b/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh @@ -0,0 +1 @@ +../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh b/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh new file mode 120000 index 000000000..3d8509fcc --- /dev/null +++ b/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh @@ -0,0 +1 @@ +../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh b/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh new file mode 120000 index 000000000..3d8509fcc --- /dev/null +++ b/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh @@ -0,0 +1 @@ +../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/020-ceph.sh b/tools/deployment/osh-infra-logging/020-ceph.sh index 677caa4bc..94dea7273 100755 --- a/tools/deployment/osh-infra-logging/020-ceph.sh +++ b/tools/deployment/osh-infra-logging/020-ceph.sh @@ -159,16 +159,22 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" + pod: replicas: mds: 1 mgr: 1 rgw: 1 +deploy: + tool: "ceph-volume" jobs: ceph_defragosds: # Execute every 15 minutes for gates diff --git a/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh b/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh new file mode 100755 index 000000000..5eba99790 --- /dev/null +++ b/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -xe +sudo df -lh +sudo lsblk +sudo mkdir -p /var/lib/openstack-helm/ceph +sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img + +#second disk for tenant-ceph +sudo mkdir -p /var/lib/openstack-helm/tenant-ceph +sudo truncate -s 10G /var/lib/openstack-helm/tenant-ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/tenant-ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop2 /var/lib/openstack-helm/tenant-ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop3 /var/lib/openstack-helm/tenant-ceph/ceph-osd-db-wal-loopbackfile.img + +# lets check the devices +sudo df -lh +sudo lsblk diff --git a/tools/deployment/tenant-ceph/030-ceph.sh b/tools/deployment/tenant-ceph/030-ceph.sh index 72c084f6b..af49111c1 100755 --- a/tools/deployment/tenant-ceph/030-ceph.sh +++ b/tools/deployment/tenant-ceph/030-ceph.sh @@ -93,11 +93,14 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" storageclass: rbd: ceph_configmap_name: ceph-etc @@ -111,6 +114,8 @@ monitoring: enabled: true ceph_mgr: port: 9283 +deploy: + tool: "ceph-volume" EOF for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do diff --git a/tools/deployment/tenant-ceph/040-tenant-ceph.sh b/tools/deployment/tenant-ceph/040-tenant-ceph.sh index 81219e4cc..76539fce3 100755 --- a/tools/deployment/tenant-ceph/040-tenant-ceph.sh +++ b/tools/deployment/tenant-ceph/040-tenant-ceph.sh @@ -131,13 +131,18 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/tenant-ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/tenant-ceph/osd/journal-one + type: bluestore + location: /dev/loop2 + block_db: + location: /dev/loop3 + size: "5GB" + block_wal: + location: /dev/loop3 + size: "2GB" mon: directory: /var/lib/openstack-helm/tenant-ceph/mon +deploy: + tool: "ceph-volume" EOF for CHART in ceph-mon ceph-osd ceph-client; do diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 94b201166..3aa00d31f 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -53,6 +53,7 @@ post-run: playbooks/osh-infra-collect-logs.yaml vars: gate_scripts: + - ./tools/deployment/multinode/019-setup-ceph-loopback-device.sh - ./tools/deployment/multinode/010-deploy-docker-registry.sh - ./tools/deployment/multinode/020-ingress.sh - ./tools/deployment/multinode/030-ceph.sh @@ -95,6 +96,7 @@ post-run: playbooks/osh-infra-collect-logs.yaml vars: gate_scripts: + - ./tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh - ./tools/deployment/tenant-ceph/010-relabel-nodes.sh - ./tools/deployment/tenant-ceph/020-ingress.sh - ./tools/deployment/tenant-ceph/030-ceph.sh @@ -132,6 +134,7 @@ vars: gate_scripts: - ./tools/deployment/osh-infra-logging/000-install-packages.sh + - ./tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh - ./tools/deployment/osh-infra-logging/010-ingress.sh - ./tools/deployment/osh-infra-logging/020-ceph.sh @@ -156,6 +159,7 @@ vars: gate_scripts: - ./tools/deployment/osh-infra-kafka/000-install-packages.sh + - ./tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh - ./tools/deployment/osh-infra-kafka/010-ingress.sh - ./tools/deployment/osh-infra-kafka/020-ceph.sh @@ -270,6 +274,7 @@ feature_gates: apparmor gate_scripts: - ./tools/deployment/apparmor/000-install-packages.sh + - ./tools/deployment/apparmor/019-setup-ceph-loopback-device.sh - ./tools/deployment/apparmor/001-setup-apparmor-profiles.sh - ./tools/deployment/apparmor/005-deploy-k8s.sh - ./tools/deployment/apparmor/015-ingress.sh @@ -305,6 +310,7 @@ feature_gates: apparmor gate_scripts: - ./tools/deployment/osh-infra-logging/000-install-packages.sh + - ./tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh - ./tools/deployment/osh-infra-logging/010-ingress.sh - ./tools/deployment/osh-infra-logging/020-ceph.sh @@ -334,6 +340,7 @@ feature_gates: apparmor gate_scripts: - ./tools/deployment/openstack-support/000-install-packages.sh + - ./tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh - ./tools/deployment/openstack-support/005-deploy-k8s.sh - ./tools/deployment/openstack-support/007-namespace-config.sh - ./tools/deployment/openstack-support/010-ingress.sh @@ -380,6 +387,7 @@ vars: gate_scripts: - ./tools/deployment/openstack-support/000-install-packages.sh + - ./tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh - ./tools/deployment/openstack-support/005-deploy-k8s.sh - ./tools/deployment/openstack-support/007-namespace-config.sh - ./tools/deployment/openstack-support/010-ingress.sh @@ -442,6 +450,7 @@ nodeset: openstack-helm-single-node vars: gate_scripts: + - ./tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh - ./tools/deployment/elastic-beats/005-deploy-k8s.sh - ./tools/deployment/elastic-beats/020-ingress.sh - ./tools/deployment/elastic-beats/030-ceph.sh