From 3591d0fa9f3211bcc707bb0fc652a8fb33545dcb Mon Sep 17 00:00:00 2001
From: Tone Zhang <tone.zhang@arm.com>
Date: Tue, 8 May 2018 07:27:56 +0000
Subject: [PATCH] Allow Kolla Ceph to deploy bluestore OSDs in Kolla-ansible

Support Kolla Ceph to deploy blustore OSDs with Kolla-ansible.

Please refer to [1] for bluestore OSD configuration

The patch includes:
1. Set Ceph OSD store type group_vars/all.yml. The default value
is "bluestore" in Rocky.

2. Make Kolla Ceph to deploy bluestore OSDs with Kolla-ansible

3. Update gate test configuration for Ceph bluestore OSD test

[1]: specs/kolla-ceph-bluestore.rst

Partially-Implements: blueprint kolla-ceph-bluestore
Depends-On: I00eaa600a5e9ad4c1ebca2eeb523bca3d7a25128
Change-Id: I14f20a00654dff32c36d078ebb9005d91a3e60b2
Signed-off-by: Tone Zhang <tone.zhang@arm.com>
---
 ansible/group_vars/all.yml                    |  4 +++
 ansible/roles/ceph/defaults/main.yml          |  4 +++
 ansible/roles/ceph/tasks/bootstrap_osds.yml   | 30 +++++++++++++++++--
 ansible/roles/ceph/tasks/reconfigure.yml      |  2 +-
 ansible/roles/ceph/tasks/start_osds.yml       |  4 ++-
 ...kolla-ceph-bluestore-b48673a85dda06d1.yaml |  6 ++++
 tests/setup_ceph_disks.sh                     | 22 ++++++++++----
 7 files changed, 62 insertions(+), 10 deletions(-)
 create mode 100644 releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 10b29c889b..10a1f33490 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -768,6 +768,10 @@ ceph_cache_rule: "cache host firstn"
 ceph_pool_pg_num: 8
 ceph_pool_pgp_num: 8
 
+# Set the store type for ceph OSD
+# Valid options are [ filestore, bluestore]
+ceph_osd_store_type: "bluestore"
+
 #####################
 # VMware support
 ######################
diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml
index 851cfdbd5f..6c312da401 100644
--- a/ansible/roles/ceph/defaults/main.yml
+++ b/ansible/roles/ceph/defaults/main.yml
@@ -46,6 +46,10 @@ ceph_client_admin_keyring_caps:
   osd: "allow *"
   mgr: "allow *"
 
+partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_BOOTSTRAP' }}"
+partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}"
+partition_name_osd_data: "{{ 'KOLLA_CEPH_BSDATA' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}"
+
 ####################
 ## Ceph_rgw_keystone
 ####################
diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml
index ff80dfc501..8a1c842e8e 100644
--- a/ansible/roles/ceph/tasks/bootstrap_osds.yml
+++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml
@@ -2,7 +2,7 @@
 - name: Looking up disks to bootstrap for Ceph OSDs
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_OSD_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -14,7 +14,7 @@
 - name: Looking up disks to bootstrap for Ceph Cache OSDs
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_cache_bootstrap }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_cache_lookup
   changed_when: osd_cache_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_cache_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_cache_lookup.stdout.split()[2] != 'SUCCESS'
@@ -58,6 +58,19 @@
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
       HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_DEV: "{{ item.1.device | default('') }}"
+      OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"
+      OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}"
+      OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}"
+      OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}"
+      OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}"
+      OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}"
+      OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label | default('') }}"
+      OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}"
+      OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}"
+      OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}"
+      OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}"
     image: "{{ ceph_osd_image_full }}"
     labels:
       BOOTSTRAP:
@@ -106,6 +119,19 @@
       OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
       OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
       HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_DEV: "{{ item.1.device | default('') }}"
+      OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}"
+      OSD_BS_PARTNUM: "{{ item.1.partition_num | default('') }}"
+      OSD_BS_BLK_DEV: "{{ item.1.bs_blk_device | default('') }}"
+      OSD_BS_BLK_LABEL: "{{ item.1.bs_blk_label | default('') }}"
+      OSD_BS_BLK_PARTNUM: "{{ item.1.bs_blk_partition_num | default('') }}"
+      OSD_BS_WAL_DEV: "{{ item.1.bs_wal_device | default('') }}"
+      OSD_BS_WAL_LABEL: "{{ item.1.bs_wal_label|default('') }}"
+      OSD_BS_WAL_PARTNUM: "{{ item.1.bs_wal_partition_num | default('') }}"
+      OSD_BS_DB_DEV: "{{ item.1.bs_db_device | default('') }}"
+      OSD_BS_DB_LABEL: "{{ item.1.bs_db_label | default('') }}"
+      OSD_BS_DB_PARTNUM: "{{ item.1.bs_db_partition_num | default('') }}"
     image: "{{ ceph_osd_image_full }}"
     labels:
       BOOTSTRAP:
diff --git a/ansible/roles/ceph/tasks/reconfigure.yml b/ansible/roles/ceph/tasks/reconfigure.yml
index 0b2e51b5c5..a4832495a9 100644
--- a/ansible/roles/ceph/tasks/reconfigure.yml
+++ b/ansible/roles/ceph/tasks/reconfigure.yml
@@ -13,7 +13,7 @@
 - name: Looking up OSDs for Ceph
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml
index 44fe8292aa..849458b85a 100644
--- a/ansible/roles/ceph/tasks/start_osds.yml
+++ b/ansible/roles/ceph/tasks/start_osds.yml
@@ -2,7 +2,7 @@
 - name: Looking up OSDs for Ceph
   command: docker exec -t kolla_toolbox sudo -E ansible localhost
     -m find_disks
-    -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
+    -a "partition_name={{ partition_name_osd_data }} match_mode='prefix' use_udev={{ kolla_ceph_use_udev }}"
   register: osd_lookup
   changed_when: osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
   failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -40,6 +40,8 @@
       OSD_ID: "{{ item.0.stdout }}"
       JOURNAL_PARTITION: "{{ item.1.journal }}"
       TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_tc_bytes }}"
+      OSD_STORETYPE: "{{ ceph_osd_store_type }}"
+      OSD_BS_FSUUID: "{{ item.1['fs_uuid'] }}"
     image: "{{ ceph_osd_image_full }}"
     name: "ceph_osd_{{ item.0.stdout }}"
     pid_mode: "host"
diff --git a/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml b/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml
new file mode 100644
index 0000000000..fa14171061
--- /dev/null
+++ b/releasenotes/notes/kolla-ceph-bluestore-b48673a85dda06d1.yaml
@@ -0,0 +1,6 @@
+---
+prelude: >
+    Since Ceph Luminous release, bluestore OSDs are recommended. Kolla Ceph
+    currently only supports filestore. Bluestore is required in Kolla Ceph.
+features:
+  - Support Kolla Ceph to deploy bluestore OSDs in Rocky release.
diff --git a/tests/setup_ceph_disks.sh b/tests/setup_ceph_disks.sh
index 66a085caa5..582894b8b4 100644
--- a/tests/setup_ceph_disks.sh
+++ b/tests/setup_ceph_disks.sh
@@ -1,13 +1,23 @@
 mkdir -p /opt/data/kolla
-dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=2048
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0.img bs=5M count=128
 LOOP=$(losetup -f)
 losetup $LOOP /opt/data/kolla/ceph-osd0.img
-parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1 1 -1
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_OSD1 1 -1
 
-dd if=/dev/zero of=/opt/data/kolla/ceph-journal0.img bs=5M count=512
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-b.img bs=5M count=2048
 LOOP=$(losetup -f)
-losetup $LOOP /opt/data/kolla/ceph-journal0.img
-parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_OSD1_J 1 -1
+losetup $LOOP /opt/data/kolla/ceph-osd0-b.img
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_B_OSD1 1 -1
 
-partprobe $LOOP
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-w.img bs=5M count=256
+LOOP=$(losetup -f)
+losetup $LOOP /opt/data/kolla/ceph-osd0-w.img
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_W_OSD1 1 -1
+
+dd if=/dev/zero of=/opt/data/kolla/ceph-osd0-d.img bs=5M count=256
+LOOP=$(losetup -f)
+losetup $LOOP /opt/data/kolla/ceph-osd0-d.img
+parted $LOOP -s -- mklabel gpt mkpart KOLLA_CEPH_OSD_BOOTSTRAP_BS_D_OSD1 1 -1
+
+partprobe