From a0a23311c3c40f631663468e1ba45d5e84790019 Mon Sep 17 00:00:00 2001
From: Sean Dague <sean.dague@samsung.com>
Date: Wed, 15 Jan 2014 15:24:30 -0500
Subject: [PATCH] updated sar options to collect more data

in order to have better data on the load state of the test nodes
we should track things beyond just cpu time. Add in load time,
process creation rates, and io rates during the tests.

also add a sar filter that makes it report on one line

reading sar input with multiple flags is somewhat problematic,
because it's tons of interspersed headers. So build something with
does a pivot filter to make it possible to get this all on one
line.

Change-Id: I8f085cedda65dfc37ad530eb97ba1fc5577314c3
---
 stack.sh            | 12 +++++--
 tools/sar_filter.py | 82 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+), 3 deletions(-)
 create mode 100755 tools/sar_filter.py

diff --git a/stack.sh b/stack.sh
index 7da41a98c8..382b75e7fc 100755
--- a/stack.sh
+++ b/stack.sh
@@ -860,11 +860,17 @@ init_service_check
 # -------
 
 # If enabled, systat has to start early to track OpenStack service startup.
-if is_service_enabled sysstat;then
+if is_service_enabled sysstat; then
+    # what we want to measure
+    # -u : cpu statitics
+    # -q : load
+    # -b : io load rates
+    # -w : process creation and context switch rates
+    SYSSTAT_OPTS="-u -q -b -w"
     if [[ -n ${SCREEN_LOGDIR} ]]; then
-        screen_it sysstat "cd ; sar -o $SCREEN_LOGDIR/$SYSSTAT_FILE $SYSSTAT_INTERVAL"
+        screen_it sysstat "cd $TOP_DIR; ./tools/sar_filter.py $SYSSTAT_OPTS -o $SCREEN_LOGDIR/$SYSSTAT_FILE $SYSSTAT_INTERVAL"
     else
-        screen_it sysstat "sar $SYSSTAT_INTERVAL"
+        screen_it sysstat "./tools/sar_filter.py $SYSSTAT_OPTS $SYSSTAT_INTERVAL"
     fi
 fi
 
diff --git a/tools/sar_filter.py b/tools/sar_filter.py
new file mode 100755
index 0000000000..ed8c19687c
--- /dev/null
+++ b/tools/sar_filter.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Samsung Electronics Corp. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+import subprocess
+import sys
+
+
+def is_data_line(line):
+    timestamp, data = parse_line(line)
+    return re.search('\d\.d', data)
+
+
+def parse_line(line):
+    m = re.search('(\d\d:\d\d:\d\d \w\w)(\s+((\S+)\s*)+)', line)
+    if m:
+        date = m.group(1)
+        data = m.group(2).rstrip()
+        return date, data
+    else:
+        return None, None
+
+
+process = subprocess.Popen(
+    "sar %s" % " ".join(sys.argv[1:]),
+    shell=True,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT)
+
+# Poll process for new output until finished
+
+start_time = ""
+header = ""
+data_line = ""
+printed_header = False
+current_ts = None
+while True:
+    nextline = process.stdout.readline()
+    if nextline == '' and process.poll() is not None:
+        break
+
+    date, data = parse_line(nextline)
+    # stop until we get to the first set of real lines
+    if not date:
+        continue
+
+    # now we eat the header lines, and only print out the header
+    # if we've never seen them before
+    if not start_time:
+        start_time = date
+        header += "%s   %s" % (date, data)
+    elif date == start_time:
+        header += "   %s" % data
+    elif not printed_header:
+        printed_header = True
+        print header
+
+    # now we know this is a data line, printing out if the timestamp
+    # has changed, and stacking up otherwise.
+    nextline = process.stdout.readline()
+    date, data = parse_line(nextline)
+    if date != current_ts:
+        current_ts = date
+        print data_line
+        data_line = "%s   %s" % (date, data)
+    else:
+        data_line += "   %s" % data
+
+    sys.stdout.flush()