[CEPH] Setup a cronjob to run OSD defrags for FileStore

Create a cron and associated script to run monthly OSD defrags. When the script runs it will switch the OSD disk to the CFQ I/O scheduler to ensure that this is a non-blocking operation for ceph. While this cron job will run monthly, it will only execute on OSDs that are HDD based with Filestore. Change-Id: I06a4679e0cbb3e065974d610606d232cde77e0b2
2018-12-19 00:39:02 -06:00 · 2018-12-19 00:39:02 -06:00 · d966085321
commit d966085321
parent cd4ec0b4b2
5 changed files with 180 additions and 0 deletions
--- a/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl
+++ b/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl
@ -0,0 +1,53 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+ARG=${1}
+
+if [ "x${ARG}" == "xcron" ]; then
+  PODS=$(kubectl get pods --namespace=${NAMESPACE} \
+  --selector=application=ceph,component=osd --field-selector=status.phase=Running \
+  '--output=jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}')
+
+  for POD in ${PODS}; do
+    kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \
+    sh -c -e "/tmp/utils-defragOSDs.sh defrag"
+  done
+fi
+
+if [ "x${ARG}" == "xdefrag" ]; then
+  OSD_PATH=$(cat /proc/mounts | awk '/ceph-/{print $2}')
+  OSD_DEVICE=$(cat /proc/mounts | awk '/ceph-/{print $1}')
+  OSD_STORE=$(cat ${OSD_PATH}/type)
+
+  ODEV=$(echo "${OSD_DEVICE}" | sed 's/\(.*[^0-9]\)[0-9]*$/\1/' | awk -F'/' '{print $3}')
+  ODEV_ROTATIONAL=$(cat /sys/block/${ODEV}/queue/rotational)
+  ODEV_SCHEDULER=$(cat /sys/block/${ODEV}/queue/scheduler)
+
+  # NOTE(supamatt): TODO implement bluestore defrag options once it's available upstream
+  if [ "${ODEV_ROTATIONAL}" -eq "1" ] && [ "x${OSD_STORE}" == "xfilestore" ]; then
+    # NOTE(supamatt): Switch to CFQ in order to not block I/O
+    echo "cfq" | tee /sys/block/${ODEV}/queue/scheduler || true
+    ionice -c 3 xfs_fsr "${OSD_DEVICE}" 2>/dev/null
+    # NOTE(supamatt): Switch back to previous IO scheduler
+    echo ${ODEV_SCHEDULER} | tee /sys/block/${ODEV}/queue/scheduler || true
+  fi
+fi
+
+exit 0
--- a/ceph-osd/templates/configmap-bin.yaml
+++ b/ceph-osd/templates/configmap-bin.yaml
@ -46,4 +46,6 @@ data:
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  utils-checkDNS.sh: |
 {{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  utils-defragOSDs.sh: |
+{{ tuple "bin/utils/_defragOSDs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
--- a/ceph-osd/templates/cronjob-defragosds.yaml
+++ b/ceph-osd/templates/cronjob-defragosds.yaml
@ -0,0 +1,105 @@
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cronjob_defragosds }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "ceph-defragosds" }}
+{{ tuple $envAll "ceph_defragosds" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/exec
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ $serviceAccountName }}
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  schedule: {{ .Values.jobs.ceph_defragosds.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.successJob }}
+  failedJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.failJob }}
+  concurrencyPolicy: {{ .Values.jobs.ceph_defragosds.concurrency.execPolicy }}
+  startingDeadlineSeconds: {{ .Values.jobs.ceph_defragosds.startingDeadlineSecs }}
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+          nodeSelector:
+            {{ .Values.labels.osd.node_selector_key }}: {{ .Values.labels.osd.node_selector_value }}
+          containers:
+          - name: {{ $serviceAccountName }}
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 12 }}
+            env:
+              - name: NAMESPACE
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.namespace
+              - name: KUBECTL_PARAM
+                value: {{ tuple $envAll "ceph" "ceph-defragosd" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
+            command: ["/tmp/utils-defragosds.sh"]
+            args: ["cron"]
+            volumeMounts:
+              - name: ceph-osd-bin
+                mountPath: /tmp/utils-defragosds.sh
+                subPath: utils-defragosds.sh
+                readOnly: true
+          restartPolicy: Never
+          hostNetwork: true
+          volumes:
+            - name: ceph-osd-bin
+              configMap:
+                name: ceph-osd-bin
+                defaultMode: 0555
+            - name: ceph-osd-etc
+              configMap:
+                name: ceph-osd-etc
+                defaultMode: 0444
+{{- end }}
--- a/ceph-osd/templates/daemonset-osd.yaml
+++ b/ceph-osd/templates/daemonset-osd.yaml
@ -272,6 +272,10 @@ spec:
              mountPath: /tmp/osd-common.sh
              subPath: osd-common.sh
              readOnly: true
+            - name: ceph-osd-bin
+              mountPath: /tmp/utils-defragOSDs.sh
+              subPath: utils-defragOSDs.sh
+              readOnly: true
            - name: ceph-osd-etc
              mountPath: /etc/ceph/storage.json
              subPath: storage.json
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@ -76,10 +76,25 @@ secrets:
  keyrings:
    osd: ceph-bootstrap-osd-keyring
    admin: ceph-client-admin-keyring
+
 network:
  public: 192.168.0.0/16
  cluster: 192.168.0.0/16

+jobs:
+  ceph_defragosds:
+    # Execute the 1st of each month
+    cron: "0 0 1 * *"
+    history:
+      # Number of successful job to keep
+      successJob: 1
+      # Number of failed job to keep
+      failJob: 1
+    concurrency:
+      # Skip new job if previous job still active
+      execPolicy: Forbid
+    startingDeadlineSecs: 60
+
 conf:
  ceph:
    global:
@ -250,3 +265,4 @@ manifests:
  daemonset_osd: true
  job_image_repo_sync: true
  helm_tests: true
+  cronjob_defragosds: true