[CEPH] update all ceph daemons startup scripts to support msgr2

This is to update all ceph daemons startup scripts as per msgr2 protocol and
also to update v2 port for mon_host config.
This also removes setting mon_addr config since we already have mon_host config.

v1 default port: 6789
V2 default port: 3300

Change-Id: I3d95edbd89f5ac8b40a34f41c1099311cee4f875

ceph-client/templates/bin/mds/_start.sh.tpl

@@ -18,11 +18,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
+  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-  if [[ ${ENDPOINT} == "" ]]; then
+             -v version=v1 -v msgr_version=v2 \
+             -v msgr2_port=${MON_PORT_V2} \
+             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
-    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
   fi
 fi
 

ceph-client/templates/bin/mgr/_start.sh.tpl

@@ -10,11 +10,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
+  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-  if [[ ${ENDPOINT} == "" ]]; then
+             -v version=v1 -v msgr_version=v2 \
+             -v msgr2_port=${MON_PORT_V2} \
+             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
-    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
   fi
 fi
 

ceph-client/templates/configmap-etc-client.yaml

@@ -22,14 +22,10 @@ limitations under the License.
 {{- if .Values.deployment.ceph }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.ceph.global.mon_addr -}}
-{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
-{{- end -}}
 
 {{- if empty .Values.conf.ceph.osd.cluster_network -}}
 {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

ceph-client/templates/deployment-mds.yaml

@@ -94,6 +94,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           ports:
             - containerPort: 6800
           livenessProbe:

ceph-client/templates/deployment-mgr.yaml

@@ -92,6 +92,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             {{- if .Values.ceph_mgr_enabled_modules }}
             - name: ENABLED_MODULES
               value: |-

ceph-mon/templates/bin/mon/_start.sh.tpl

@@ -12,7 +12,7 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
+  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
              -v version=v1 -v msgr_version=v2 \
              -v msgr2_port=${MON_PORT_V2} \
              '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
@@ -54,9 +54,9 @@ function get_mon_config {
   while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
     # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
     if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
+        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.metadata.name}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
     else
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
+        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.spec.nodeName}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
     fi
     (( timeout-- ))
     sleep 1
@@ -99,7 +99,7 @@ else
   # no mons are up and running yet
   timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
   ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
-  timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
+  timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT_V2}" || true
 fi
 
 # start MON
@@ -110,4 +110,4 @@ exec /usr/bin/ceph-mon \
   -d \
   -i ${MON_NAME} \
   --mon-data "${MON_DATA_DIR}" \
-  --public-addr "${MON_IP}:${MON_PORT}"
+  --public-addr "${MON_IP}:${MON_PORT_V2}"

ceph-mon/templates/bin/utils/_checkDNS.sh.tpl

@@ -17,17 +17,14 @@ limitations under the License.
 */}}
 
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
-ENDPOINT=$1
+ENDPOINT="{$1}"
 
 function check_mon_dns () {
   GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
 
-  if [[ ${ENDPOINT} == "up" ]]; then
+  if [[ "${ENDPOINT}" == "up" ]]; then
-    # If DNS is working, we simply clean up the ${CEPH_CONF} file
+    echo "If DNS is working, we are good here"
-    if [[ ${GREP_CMD} == "" ]]; then
+  elif [[ "${ENDPOINT}" != "" ]]; then
-      sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
-    fi
-  elif [[ ${ENDPOINT} != "" ]]; then
     if [[ ${GREP_CMD} != "" ]]; then
       # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
       sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1

ceph-mon/templates/configmap-etc.yaml

@@ -22,7 +22,7 @@ limitations under the License.
 {{- if .Values.deployment.ceph }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $monHost := tuple "ceph_mon" "discovery" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 

ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl

@@ -44,12 +44,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
+  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-  if [[ ${ENDPOINT} == "" ]]; then
+             -v version=v1 -v msgr_version=v2 \
-    # No endpoints are available, just copy ceph.conf as-is
+             -v msgr2_port=${MON_PORT_V2} \
+             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
-    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
   fi
 fi
 

ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl

@@ -44,12 +44,14 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then
   echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon"
   exit 1
 else
-  ENDPOINT=$(kubectl get endpoints ceph-mon -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} '/"ip"/{print $4":"port}' | paste -sd',')
+  ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \
-  if [[ ${ENDPOINT} == "" ]]; then
+             -v version=v1 -v msgr_version=v2 \
-    # No endpoints are available, just copy ceph.conf as-is
+             -v msgr2_port=${MON_PORT_V2} \
+             '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',')
+  if [[ "${ENDPOINT}" == "" ]]; then
     /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true
   else
-    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" || true
+    /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true
   fi
 fi
 

ceph-osd/templates/bin/utils/_checkDNS.sh.tpl

@@ -17,17 +17,14 @@ limitations under the License.
 */}}
 
 : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}"
-ENDPOINT=$1
+ENDPOINT="{$1}"
 
 function check_mon_dns () {
   GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF})
 
-  if [[ ${ENDPOINT} == "up" ]]; then
+  if [[ "${ENDPOINT}" == "up" ]]; then
-    # If DNS is working, we simply clean up the ${CEPH_CONF} file
+    echo "If DNS is working, we are good here"
-    if [[ ${GREP_CMD} == "" ]]; then
+  elif [[ "${ENDPOINT}" != "" ]]; then
-      sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" > /dev/null 2>&1
-    fi
-  elif [[ ${ENDPOINT} != "" ]]; then
     if [[ ${GREP_CMD} != "" ]]; then
       # No DNS, write CEPH MONs IPs into ${CEPH_CONF}
       sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1

ceph-osd/templates/configmap-etc.yaml

@@ -20,15 +20,10 @@ limitations under the License.
 {{- with $envAll }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.ceph.global.mon_addr -}}
-{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
-{{- end -}}
-
 {{- if empty .Values.conf.ceph.global.fsid -}}
 {{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
 {{- end -}}

ceph-osd/templates/daemonset-osd.yaml

@@ -79,6 +79,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -163,6 +165,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           command:
             - /tmp/osd-init.sh
           volumeMounts:
@@ -260,6 +264,8 @@ spec:
                   fieldPath: metadata.namespace
             - name: MON_PORT
               value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: MON_PORT_V2
+              value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
           command:
             - /tmp/osd-start.sh
           lifecycle:

ceph-osd/values.yaml

@@ -332,6 +332,8 @@ endpoints:
     port:
       mon:
         default: 6789
+      mon_msgr2:
+        default: 3300
 
 manifests:
   configmap_bin: true

ceph-provisioners/templates/configmap-etc-client.yaml

@@ -22,14 +22,10 @@ limitations under the License.
 {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.ceph.global.mon_addr -}}
-{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
-{{- end -}}
 
 {{- if empty .Values.conf.ceph.osd.cluster_network -}}
 {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

ceph-provisioners/values.yaml

@@ -287,6 +287,9 @@ endpoints:
     port:
       mon:
         default: 6789
+      mon_msgr2:
+        default: 3300
+
 
 manifests:
   configmap_bin: true

ceph-rgw/templates/configmap-etc-client.yaml

@@ -22,14 +22,10 @@ limitations under the License.
 {{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
 
 {{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
 {{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.ceph.global.mon_addr -}}
-{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
-{{- end -}}
 
 {{- if empty .Values.conf.ceph.osd.cluster_network -}}
 {{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}

ceph-rgw/values.yaml

@@ -547,6 +547,9 @@ endpoints:
     port:
       mon:
         default: 6789
+      mon_msgr2:
+        default: 3300
+
   kube_dns:
     namespace: kube-system
     name: kubernetes-dns

helm-toolkit/templates/manifests/_ceph-storageclass.tpl

@@ -88,7 +88,7 @@ examples:
 {{- $envAll := index . "envAll" -}}
 {{- $monHost := $envAll.Values.conf.ceph.global.mon_host -}}
 {{- if empty $monHost -}}
-{{- $monHost = tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
+{{- $monHost = tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}}
 {{- end -}}
 {{- $storageclassData := index . "storageclass_data" -}}
 ---