72a8be60cd
We can see at-a-glance memory usage during the run with dstat but we have no way to break that down into an overview of where memory is going. This adds a peer-service to dstat that records snapshots of the system during peak memory usage. It checks periodically if there is less memory available than before and, if so, records the running processes and vm overview. The intent is to add logic into the verify-pipeline jobs to use this report and send statistics on peak memory usage to statsd [1]. We can then build a picture of memory-usage growth over time. This type of report would have allowed better insight into issues such as introduced by Idf3a3a914b54779172776822710b3e52e751b1d1 where memory-usage jumped dramatically after switching to pip versions of libraries. Tracking details of memory usage is going to be an important part of future development. [1] http://graphite.openstack.org/ Change-Id: I4b0a8f382dcaa09331987ab84a68546ec29cbc18
39 lines
819 B
Bash
39 lines
819 B
Bash
#!/bin/bash
|
|
#
|
|
# lib/dstat
|
|
# Functions to start and stop dstat
|
|
|
|
# Dependencies:
|
|
#
|
|
# - ``functions`` file
|
|
|
|
# ``stack.sh`` calls the entry points in this order:
|
|
#
|
|
# - start_dstat
|
|
# - stop_dstat
|
|
|
|
# Save trace setting
|
|
XTRACE=$(set +o | grep xtrace)
|
|
set +o xtrace
|
|
|
|
# start_dstat() - Start running processes, including screen
|
|
function start_dstat {
|
|
# A better kind of sysstat, with the top process per time slice
|
|
DSTAT_OPTS="-tcmndrylpg --top-cpu-adv --top-io-adv"
|
|
run_process dstat "dstat $DSTAT_OPTS"
|
|
|
|
# To enable peakmem_tracker add:
|
|
# enable_service peakmem_tracker
|
|
# to your localrc
|
|
run_process peakmem_tracker "$TOP_DIR/tools/peakmem_tracker.sh"
|
|
}
|
|
|
|
# stop_dstat() stop dstat process
|
|
function stop_dstat {
|
|
stop_process dstat
|
|
stop_process peakmem_tracker
|
|
}
|
|
|
|
# Restore xtrace
|
|
$XTRACE
|