diff --git a/lib/dstat b/lib/dstat index 4b22752ced..f11bfa55c0 100644 --- a/lib/dstat +++ b/lib/dstat @@ -21,11 +21,17 @@ function start_dstat { # A better kind of sysstat, with the top process per time slice DSTAT_OPTS="-tcmndrylpg --top-cpu-adv --top-io-adv" run_process dstat "dstat $DSTAT_OPTS" + + # To enable peakmem_tracker add: + # enable_service peakmem_tracker + # to your localrc + run_process peakmem_tracker "$TOP_DIR/tools/peakmem_tracker.sh" } # stop_dstat() stop dstat process function stop_dstat { stop_process dstat + stop_process peakmem_tracker } # Restore xtrace diff --git a/tools/peakmem_tracker.sh b/tools/peakmem_tracker.sh new file mode 100755 index 0000000000..0d5728a538 --- /dev/null +++ b/tools/peakmem_tracker.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -o errexit + +# time to sleep between checks +SLEEP_TIME=20 + +# MemAvailable is the best estimation and has built-in heuristics +# around reclaimable memory. However, it is not available until 3.14 +# kernel (i.e. Ubuntu LTS Trusty misses it). In that case, we fall +# back to free+buffers+cache as the available memory. +USE_MEM_AVAILBLE=0 +if grep -q '^MemAvailable:' /proc/meminfo; then + USE_MEM_AVAILABLE=1 +fi + +function get_mem_available { + if [[ $USE_MEM_AVAILABLE -eq 1 ]]; then + awk '/^MemAvailable:/ {print $2}' /proc/meminfo + else + awk '/^MemFree:/ {free=$2} + /^Buffers:/ {buffers=$2} + /^Cached:/ {cached=$2} + END { print free+buffers+cached }' /proc/meminfo + fi +} + +# whenever we see less memory available than last time, dump the +# snapshot of current usage; i.e. checking the latest entry in the +# file will give the peak-memory usage +function tracker { + local low_point=$(get_mem_available) + while [ 1 ]; do + + local mem_available=$(get_mem_available) + + if [[ $mem_available -lt $low_point ]]; then + low_point=$mem_available + echo "[[[" + date + echo "---" + # always available greppable output; given difference in + # meminfo output as described above... + echo "peakmem_tracker low_point: $mem_available" + echo "---" + cat /proc/meminfo + echo "---" + # would hierarchial view be more useful (-H)? output is + # not sorted by usage then, however, and the first + # question is "what's using up the memory" + # + # there are a lot of kernel threads, especially on a 8-cpu + # system. do a best-effort removal to improve + # signal/noise ratio of output. + ps --sort=-pmem -eo pid:10,pmem:6,rss:15,ppid:10,cputime:10,nlwp:8,wchan:25,args:100 | + grep -v ']$' + echo "]]]" + fi + + sleep $SLEEP_TIME + done +} + +function usage { + echo "Usage: $0 [-x] [-s N]" 1>&2 + exit 1 +} + +while getopts ":s:x" opt; do + case $opt in + s) + SLEEP_TIME=$OPTARG + ;; + x) + set -o xtrace + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +tracker