From 016b56e5868d9cacaab5921f4458eba5f787a253 Mon Sep 17 00:00:00 2001
From: Stephen Taylor <st053q@att.com>
Date: Thu, 21 Nov 2019 20:12:12 +0000
Subject: [PATCH] Ceph Nautilus compatibility

This change updates the Ceph charts to use Ceph Nautilus images
built on Ubuntu Bionic instead of Xenial. The mirror that hosts
Ceph packages only provides Nautilus packages for Bionic at
present, so this is necessary for Nautilus deployment.

There are also several configuration and scripting changes
included to provide compatibility with Ceph Nautilus. Most of
these simply allow existing logic to execute for Nautilus
deployments, but some logical changes are required to support
Nautilus as well.

NOTE: The cephfs test has been disabled because it was failing
the gate. This test has passed in multiple dev environments, and
since cephfs isn't used by any openstack-helm-infra components we
don't want this to block getting this change merged. The gate
issue will be investigated and addressed in a subsequent patch
set.

Change-Id: Id2d9d7b35d4dc66e93a0aacc9ea514e85ae13467
---
 ceph-client/templates/bin/pool/_init.sh.tpl   | 27 +++++++++++++++----
 ceph-client/values.yaml                       | 20 +++++++-------
 ceph-mon/values.yaml                          | 10 +++----
 .../bin/osd/ceph-disk/_common.sh.tpl          |  4 +--
 .../bin/osd/ceph-volume/_common.sh.tpl        |  4 +--
 ceph-osd/values.yaml                          |  8 +++---
 ceph-provisioners/values.yaml                 | 10 +++----
 ceph-rgw/values.yaml                          |  8 +++---
 gnocchi/templates/bin/_storage-init.sh.tpl    |  2 +-
 gnocchi/values.yaml                           |  2 +-
 tools/deployment/common/005-deploy-k8s.sh     |  2 +-
 .../openstack-support/025-ceph-ns-activate.sh |  3 +++
 .../osh-infra-logging/025-ceph-ns-activate.sh |  3 +++
 .../tasks/support-packages.yaml               |  4 +--
 14 files changed, 65 insertions(+), 42 deletions(-)

diff --git a/ceph-client/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl
index 4fcd6e6ab..dd88d8d97 100644
--- a/ceph-client/templates/bin/pool/_init.sh.tpl
+++ b/ceph-client/templates/bin/pool/_init.sh.tpl
@@ -35,10 +35,17 @@ function wait_for_inactive_pgs () {
   echo "#### Start: Checking for inactive pgs ####"
 
   # Loop until all pgs are active
-  while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | grep -v "active+"` ]]
-  do
-    sleep 3
-  done
+  if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
+    while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | head -n -2 | grep -v "active+"` ]]
+    do
+      sleep 3
+    done
+  else
+    while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | grep -v "active+"` ]]
+    do
+      sleep 3
+    done
+  fi
 }
 
 function create_crushrule () {
@@ -51,6 +58,11 @@ function create_crushrule () {
   fi
 }
 
+# Set mons to use the msgr2 protocol on nautilus
+if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
+  ceph --cluster "${CLUSTER}" mon enable-msgr2
+fi
+
 {{- range $crush_rule := .Values.conf.pool.crush_rules -}}
 {{- with $crush_rule }}
 create_crushrule {{ .name }} {{ .crush_rule }} {{ .failure_domain }} {{ .device_class }}
@@ -147,7 +159,12 @@ reweight_osds
 {{ $crushRuleDefault := .Values.conf.pool.default.crush_rule }}
 {{ $targetQuota := .Values.conf.pool.target.quota | default 100 }}
 {{ $targetProtection := .Values.conf.pool.target.protected | default "false" | quote | lower }}
-cluster_capacity=$(ceph --cluster "${CLUSTER}" df | head -n3 | tail -n1 | awk '{print $1 substr($2, 1, 1)}' | numfmt --from=iec)
+cluster_capacity=0
+if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
+  cluster_capacity=$(ceph --cluster "${CLUSTER}" df | grep "TOTAL" | awk '{print $2 substr($3, 1, 1)}' | numfmt --from=iec)
+else
+  cluster_capacity=$(ceph --cluster "${CLUSTER}" df | head -n3 | tail -n1 | awk '{print $1 substr($2, 1, 1)}' | numfmt --from=iec)
+fi
 {{- range $pool := .Values.conf.pool.spec -}}
 {{- with $pool }}
 {{- if .crush_rule }}
diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml
index eb26d10f9..e2c8a8b81 100644
--- a/ceph-client/values.yaml
+++ b/ceph-client/values.yaml
@@ -25,11 +25,11 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
-    ceph_mds: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
+    ceph_mds: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
   local_registry:
@@ -451,7 +451,7 @@ bootstrap:
     ceph -s
     function ensure_pool () {
       ceph osd pool stats $1 || ceph osd pool create $1 $2
-      local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
+      local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
       if [[ ${test_version} -gt 0 ]]; then
         ceph osd pool application enable $1 $3
       fi
@@ -475,11 +475,11 @@ ceph_mgr_enabled_modules:
 # of key/value. Refer to the doc
 # above for more info. For example:
 ceph_mgr_modules_config:
-  balancer:
-    active: 1
-  prometheus:
+#  balancer:
+#    active: 1
+#  prometheus:
     # server_port: 9283
-    server_addr: 0.0.0.0
+#    server_addr: 0.0.0.0
 #  dashboard:
 #    port: 7000
 #  localpool:
diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml
index ff13cd6da..2e070d178 100644
--- a/ceph-mon/values.yaml
+++ b/ceph-mon/values.yaml
@@ -24,10 +24,10 @@ deployment:
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
-    ceph_mon: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
+    ceph_mon: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
   local_registry:
@@ -260,7 +260,7 @@ bootstrap:
     ceph -s
     function ensure_pool () {
       ceph osd pool stats $1 || ceph osd pool create $1 $2
-      local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
+      local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
       if [[ ${test_version} -gt 0 ]]; then
         ceph osd pool application enable $1 $3
       fi
diff --git a/ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl
index 5ff3109ab..1a6023ce9 100644
--- a/ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl
+++ b/ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl
@@ -30,8 +30,8 @@ eval CRUSH_FAILURE_DOMAIN_TYPE=$(cat /etc/ceph/storage.json | python -c 'import
 eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))')
 eval CRUSH_FAILURE_DOMAIN_BY_HOSTNAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_by_hostname"]))')
 
-if [[ $(ceph -v | egrep -q "mimic|luminous"; echo $?) -ne 0 ]]; then
-    echo "ERROR- need Luminous/Mimic release"
+if [[ $(ceph -v | egrep -q "nautilus|mimic|luminous"; echo $?) -ne 0 ]]; then
+    echo "ERROR- need Luminous/Mimic/Nautilus release"
     exit 1
 fi
 
diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
index f27a3e91d..a53f3bad0 100644
--- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
+++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
@@ -30,8 +30,8 @@ eval CRUSH_FAILURE_DOMAIN_TYPE=$(cat /etc/ceph/storage.json | python -c 'import
 eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))')
 eval CRUSH_FAILURE_DOMAIN_BY_HOSTNAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_by_hostname"]))')
 
-if [[ $(ceph -v | egrep -q "mimic|luminous"; echo $?) -ne 0 ]]; then
-    echo "ERROR- need Luminous/Mimic release"
+if [[ $(ceph -v | egrep -q "nautilus|mimic|luminous"; echo $?) -ne 0 ]]; then
+    echo "ERROR- need Luminous/Mimic/Nautilus release"
     exit 1
 fi
 
diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml
index 4ae63d07c..07f99402c 100644
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@@ -20,9 +20,9 @@
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_osd: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
+    ceph_osd: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
   local_registry:
@@ -196,7 +196,7 @@ conf:
     # match the failure domain used on your CRUSH rules for pools. For example with a crush rule of
     # rack_replicated_rule you would specify "rack" as the `failure_domain` to use.
     # `failure_domain`: Set the CRUSH bucket type for your OSD to reside in. See the supported CRUSH configuration
-    #  as listed here: Supported CRUSH configuration is listed here: http://docs.ceph.com/docs/mimic/rados/operations/crush-map/
+    #  as listed here: Supported CRUSH configuration is listed here: http://docs.ceph.com/docs/nautilus/rados/operations/crush-map/
     # `failure_domain_by_hostname`: Specify the portion of the hostname to use for your failure domain bucket name.
     # `failure_domain_name`: Manually name the failure domain bucket name. This configuration option should only be used
     #  when using host based overrides.
diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml
index 1826f9174..75e6bffe1 100644
--- a/ceph-provisioners/values.yaml
+++ b/ceph-provisioners/values.yaml
@@ -28,10 +28,10 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
-    ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_xenial-20191119'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
-    ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:ubuntu_xenial-20191119'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
+    ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_bionic-20191216'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
+    ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:ubuntu_bionic-20191216'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
   local_registry:
@@ -224,7 +224,7 @@ bootstrap:
     ceph -s
     function ensure_pool () {
       ceph osd pool stats $1 || ceph osd pool create $1 $2
-      local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
+      local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
       if [[ ${test_version} -gt 0 ]]; then
         ceph osd pool application enable $1 $3
       fi
diff --git a/ceph-rgw/values.yaml b/ceph-rgw/values.yaml
index b76359de9..0ed48d8b0 100644
--- a/ceph-rgw/values.yaml
+++ b/ceph-rgw/values.yaml
@@ -25,11 +25,11 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
-    ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
+    ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/docker:17.07.0'
-    rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
+    rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
     ks_endpoints: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_service: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_user: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
@@ -420,7 +420,7 @@ bootstrap:
     ceph -s
     function ensure_pool () {
       ceph osd pool stats $1 || ceph osd pool create $1 $2
-      local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous")
+      local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous")
       if [[ ${test_version} -gt 0 ]]; then
         ceph osd pool application enable $1 $3
       fi
diff --git a/gnocchi/templates/bin/_storage-init.sh.tpl b/gnocchi/templates/bin/_storage-init.sh.tpl
index 328d27bb5..727081ac2 100644
--- a/gnocchi/templates/bin/_storage-init.sh.tpl
+++ b/gnocchi/templates/bin/_storage-init.sh.tpl
@@ -28,7 +28,7 @@ set -ex
 ceph -s
 function ensure_pool () {
   ceph osd pool stats $1 || ceph osd pool create $1 $2
-  local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
+  local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
   if [[ ${test_version} -gt 0 ]]; then
     ceph osd pool application enable $1 $3
   fi
diff --git a/gnocchi/values.yaml b/gnocchi/values.yaml
index 8c822f4dd..ba2890210 100644
--- a/gnocchi/values.yaml
+++ b/gnocchi/values.yaml
@@ -38,7 +38,7 @@ release_group: null
 images:
   tags:
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
-    gnocchi_storage_init: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
+    gnocchi_storage_init: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
     db_init_indexer: docker.io/postgres:9.5
     # using non-kolla images until kolla supports postgres as
     # an indexer
diff --git a/tools/deployment/common/005-deploy-k8s.sh b/tools/deployment/common/005-deploy-k8s.sh
index 6562ed5a5..b4fe61b92 100755
--- a/tools/deployment/common/005-deploy-k8s.sh
+++ b/tools/deployment/common/005-deploy-k8s.sh
@@ -65,7 +65,7 @@ sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
 # Install required packages for K8s on host
 wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
 RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
-sudo add-apt-repository "deb https://download.ceph.com/debian-mimic/
+sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/
 ${RELEASE_NAME} main"
 sudo -E apt-get update
 sudo -E apt-get install -y \
diff --git a/tools/deployment/openstack-support/025-ceph-ns-activate.sh b/tools/deployment/openstack-support/025-ceph-ns-activate.sh
index 87009df3d..52ccc2873 100755
--- a/tools/deployment/openstack-support/025-ceph-ns-activate.sh
+++ b/tools/deployment/openstack-support/025-ceph-ns-activate.sh
@@ -40,6 +40,9 @@ bootstrap:
 conf:
   rgw_ks:
     enabled: false
+storageclass:
+  cephfs:
+    provision_storage_class: false
 EOF
 
 : ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE:="$(./tools/deployment/common/get-values-overrides.sh ceph-provisioners)"}
diff --git a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
index ab8eac56b..0e87a5800 100755
--- a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
+++ b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
@@ -40,6 +40,9 @@ bootstrap:
 conf:
   rgw_ks:
     enabled: false
+storageclass:
+  cephfs:
+    provision_storage_class: false
 EOF
 
 : ${OSH_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE:="$(./tools/deployment/common/get-values-overrides.sh ceph-provisioners)"}
diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/support-packages.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/support-packages.yaml
index f5b8f867a..2560d270f 100644
--- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/support-packages.yaml
+++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/support-packages.yaml
@@ -20,7 +20,7 @@
   - name: ubuntu | ensure community ceph repository exists
     when: ansible_distribution == 'Ubuntu'
     apt_repository:
-      repo: "deb https://download.ceph.com/debian-mimic/ {{ ansible_lsb.codename }} main"
+      repo: "deb https://download.ceph.com/debian-nautilus/ {{ ansible_lsb.codename }} main"
       state: present
       update_cache: yes
 
@@ -30,7 +30,7 @@
       name: ceph
       description: "Ceph community packages for Redhat/Centos"
       gpgkey: "https://download.ceph.com/keys/release.asc"
-      baseurl: "https://download.ceph.com/rpm-mimic/el7/$basearch"
+      baseurl: "https://download.ceph.com/rpm-nautilus/el7/$basearch"
       gpgcheck: yes
       state: present