Ceph-OSD: Add log tailer to ceph-osd

This change adds in a log tailer script in a pod that tails the ceph
logs, periodicly truncating them.

Change-Id: Idbe03d4123f86b1b88e277fea6d13f58104f94b0
Signed-off-by: Pete Birley <pete@port.direct>

ceph-osd/templates/bin/osd/_directory.sh.tpl

@@ -76,6 +76,9 @@ fi
 mkdir -p /etc/forego/"${CLUSTER}"
 echo "" > /etc/forego/"${CLUSTER}"/Procfile
 
+# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
+echo "${OSD_ID}" > /tmp/osd-id
+
 for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do
   OSD_PATH="$OSD_PATH_BASE-$OSD_ID/"
   OSD_KEYRING="${OSD_PATH%/}/keyring"

ceph-osd/templates/bin/osd/_log-tail.sh.tpl

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -ex
+
+osd_id_file="/tmp/osd-id"
+
+function wait_for_file() {
+  local file="$1"; shift
+  local wait_seconds="${1:-30}"; shift
+
+  until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do
+    sleep 1
+  done
+
+  ((++wait_seconds))
+}
+wait_for_file "${osd_id_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}"
+
+log_file="/var/log/ceph/${DAEMON_NAME}.$(cat "${osd_id_file}").log"
+wait_for_file "${log_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}"
+
+function tail_file () {
+  while true; do
+    tail --retry -f "${log_file}"
+  done
+}
+
+function truncate_log () {
+  while true; do
+    sleep ${TRUNCATE_PERIOD}
+    if [[ -f ${log_file} ]] ; then
+      truncate -s "${TRUNCATE_SIZE}" "${log_file}"
+    fi
+  done
+}
+
+tail_file &
+truncate_log &
+
+wait -n

ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl

@@ -107,6 +107,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
   chown -R ceph. ${OSD_PATH};
 fi
 
+# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
+echo "${OSD_ID}" > /tmp/osd-id
+
 if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then
   chown -R ceph. /var/lib/ceph/journal
   ceph-osd \

ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl

@@ -64,6 +64,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
   chown -R ceph. ${OSD_PATH};
 fi
 
+# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
+echo "${OSD_ID}" > /tmp/osd-id
+
 exec /usr/bin/ceph-osd \
     --cluster ${CLUSTER} \
     ${CEPH_OSD_OPTIONS} \

ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl

@@ -128,6 +128,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
   chown -R ceph. ${OSD_PATH};
 fi
 
+# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
+echo "${OSD_ID}" > /tmp/osd-id
+
 if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then
   chown -R ceph. /var/lib/ceph/journal
   ceph-osd \

ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl

@@ -102,6 +102,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
   chown -R ceph. ${OSD_PATH};
 fi
 
+# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
+echo "${OSD_ID}" > /tmp/osd-id
+
 exec /usr/bin/ceph-osd \
     --cluster ${CLUSTER} \
     ${CEPH_OSD_OPTIONS} \

ceph-osd/templates/configmap-bin.yaml

@@ -32,6 +32,8 @@ data:
 {{- end }}
   osd-start.sh: |
 {{ tuple "bin/osd/_start.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  log-tail.sh: |
+{{ tuple "bin/osd/_log-tail.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   osd-directory-ceph-disk.sh: |
 {{ tuple "bin/osd/_directory.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   osd-block-ceph-disk.sh: |

ceph-osd/templates/daemonset-osd.yaml

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-
 {{- define "ceph.osd.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -236,6 +235,30 @@ spec:
               mountPath: /var/log/ceph
               readOnly: false
       containers:
+        - name: log-runner
+{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "osd" "container" "log_runner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: DAEMON_NAME
+              value: "ceph-osd"
+            - name: TRUNCATE_SIZE
+              value: {{ .Values.logging.truncate.size | quote }}
+            - name: TRUNCATE_PERIOD
+              value: {{ .Values.logging.truncate.period | quote }}
+            - name: WAIT_FOR_OSD_ID_TIMEOUT
+              value: {{ .Values.logging.osd_id.timeout | quote }}
+          command:
+            - /tmp/log-tail.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: ceph-osd-bin
+              mountPath: /tmp/log-tail.sh
+              subPath: log-tail.sh
+              readOnly: true
+            - name: pod-var-log
+              mountPath: /var/log/ceph
+              readOnly: false
         - name: ceph-osd-default
 {{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}

ceph-osd/values.yaml

@@ -69,6 +69,9 @@ pod:
           runAsUser: 0
           privileged: true
           readOnlyRootFilesystem: true
+        log_runner:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
     bootstrap:
       pod:
         runAsUser: 65534
@@ -303,6 +306,13 @@ dependencies:
         - endpoint: internal
           service: ceph_mon
 
+logging:
+  truncate:
+    size: 0
+    period: 3600
+  osd_id:
+    timeout: 300
+
 bootstrap:
   enabled: true
   script: |