diff --git a/ceph-osd/templates/bin/osd/_directory.sh.tpl b/ceph-osd/templates/bin/osd/_directory.sh.tpl index 74432ab14..1e0b49265 100644 --- a/ceph-osd/templates/bin/osd/_directory.sh.tpl +++ b/ceph-osd/templates/bin/osd/_directory.sh.tpl @@ -76,6 +76,9 @@ fi mkdir -p /etc/forego/"${CLUSTER}" echo "" > /etc/forego/"${CLUSTER}"/Procfile +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do OSD_PATH="$OSD_PATH_BASE-$OSD_ID/" OSD_KEYRING="${OSD_PATH%/}/keyring" diff --git a/ceph-osd/templates/bin/osd/_log-tail.sh.tpl b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl new file mode 100644 index 000000000..a6b3edd10 --- /dev/null +++ b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl @@ -0,0 +1,39 @@ +#!/bin/bash +set -ex + +osd_id_file="/tmp/osd-id" + +function wait_for_file() { + local file="$1"; shift + local wait_seconds="${1:-30}"; shift + + until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do + sleep 1 + done + + ((++wait_seconds)) +} +wait_for_file "${osd_id_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}" + +log_file="/var/log/ceph/${DAEMON_NAME}.$(cat "${osd_id_file}").log" +wait_for_file "${log_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}" + +function tail_file () { + while true; do + tail --retry -f "${log_file}" + done +} + +function truncate_log () { + while true; do + sleep ${TRUNCATE_PERIOD} + if [[ -f ${log_file} ]] ; then + truncate -s "${TRUNCATE_SIZE}" "${log_file}" + fi + done +} + +tail_file & +truncate_log & + +wait -n diff --git a/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl index 0773e3168..f00beb04a 100644 --- a/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl @@ -107,6 +107,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then chown -R ceph. ${OSD_PATH}; fi +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then chown -R ceph. /var/lib/ceph/journal ceph-osd \ diff --git a/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl index f51f7fa1b..51e6815c6 100644 --- a/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl @@ -64,6 +64,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then chown -R ceph. ${OSD_PATH}; fi +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + exec /usr/bin/ceph-osd \ --cluster ${CLUSTER} \ ${CEPH_OSD_OPTIONS} \ diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl index bc657ec01..27c94ce3a 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl @@ -128,6 +128,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then chown -R ceph. ${OSD_PATH}; fi +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then chown -R ceph. /var/lib/ceph/journal ceph-osd \ diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl index 54686f8af..3bc8e0d22 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl @@ -102,6 +102,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then chown -R ceph. ${OSD_PATH}; fi +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + exec /usr/bin/ceph-osd \ --cluster ${CLUSTER} \ ${CEPH_OSD_OPTIONS} \ diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml index 32eedcdcf..d8870eace 100644 --- a/ceph-osd/templates/configmap-bin.yaml +++ b/ceph-osd/templates/configmap-bin.yaml @@ -32,6 +32,8 @@ data: {{- end }} osd-start.sh: | {{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + log-tail.sh: | +{{ tuple "bin/osd/_log-tail.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd-directory-ceph-disk.sh: | {{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd-block-ceph-disk.sh: | diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 047248c94..108df1a79 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */}} - {{- define "ceph.osd.daemonset" }} {{- $daemonset := index . 0 }} {{- $configMapName := index . 1 }} @@ -236,6 +235,30 @@ spec: mountPath: /var/log/ceph readOnly: false containers: + - name: log-runner +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "log_runner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DAEMON_NAME + value: "ceph-osd" + - name: TRUNCATE_SIZE + value: {{ .Values.logging.truncate.size | quote }} + - name: TRUNCATE_PERIOD + value: {{ .Values.logging.truncate.period | quote }} + - name: WAIT_FOR_OSD_ID_TIMEOUT + value: {{ .Values.logging.osd_id.timeout | quote }} + command: + - /tmp/log-tail.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-osd-bin + mountPath: /tmp/log-tail.sh + subPath: log-tail.sh + readOnly: true + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false - name: ceph-osd-default {{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 54a41af8b..985e379cf 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -69,6 +69,9 @@ pod: runAsUser: 0 privileged: true readOnlyRootFilesystem: true + log_runner: + runAsUser: 0 + readOnlyRootFilesystem: true bootstrap: pod: runAsUser: 65534 @@ -303,6 +306,13 @@ dependencies: - endpoint: internal service: ceph_mon +logging: + truncate: + size: 0 + period: 3600 + osd_id: + timeout: 300 + bootstrap: enabled: true script: |