updated sar options to collect more data

in order to have better data on the load state of the test nodes
we should track things beyond just cpu time. Add in load time,
process creation rates, and io rates during the tests.

also add a sar filter that makes it report on one line

reading sar input with multiple flags is somewhat problematic,
because it's tons of interspersed headers. So build something with
does a pivot filter to make it possible to get this all on one
line.

Change-Id: I8f085cedda65dfc37ad530eb97ba1fc5577314c3

stack.sh

@@ -860,11 +860,17 @@ init_service_check
 # -------
 
 # If enabled, systat has to start early to track OpenStack service startup.
-if is_service_enabled sysstat;then
+if is_service_enabled sysstat; then
+    # what we want to measure
+    # -u : cpu statitics
+    # -q : load
+    # -b : io load rates
+    # -w : process creation and context switch rates
+    SYSSTAT_OPTS="-u -q -b -w"
     if [[ -n ${SCREEN_LOGDIR} ]]; then
-        screen_it sysstat "cd ; sar -o $SCREEN_LOGDIR/$SYSSTAT_FILE $SYSSTAT_INTERVAL"
+        screen_it sysstat "cd $TOP_DIR; ./tools/sar_filter.py $SYSSTAT_OPTS -o $SCREEN_LOGDIR/$SYSSTAT_FILE $SYSSTAT_INTERVAL"
     else
-        screen_it sysstat "sar $SYSSTAT_INTERVAL"
+        screen_it sysstat "./tools/sar_filter.py $SYSSTAT_OPTS $SYSSTAT_INTERVAL"
     fi
 fi
 

tools/sar_filter.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Samsung Electronics Corp. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+import subprocess
+import sys
+
+
+def is_data_line(line):
+    timestamp, data = parse_line(line)
+    return re.search('\d\.d', data)
+
+
+def parse_line(line):
+    m = re.search('(\d\d:\d\d:\d\d \w\w)(\s+((\S+)\s*)+)', line)
+    if m:
+        date = m.group(1)
+        data = m.group(2).rstrip()
+        return date, data
+    else:
+        return None, None
+
+
+process = subprocess.Popen(
+    "sar %s" % " ".join(sys.argv[1:]),
+    shell=True,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT)
+
+# Poll process for new output until finished
+
+start_time = ""
+header = ""
+data_line = ""
+printed_header = False
+current_ts = None
+while True:
+    nextline = process.stdout.readline()
+    if nextline == '' and process.poll() is not None:
+        break
+
+    date, data = parse_line(nextline)
+    # stop until we get to the first set of real lines
+    if not date:
+        continue
+
+    # now we eat the header lines, and only print out the header
+    # if we've never seen them before
+    if not start_time:
+        start_time = date
+        header += "%s   %s" % (date, data)
+    elif date == start_time:
+        header += "   %s" % data
+    elif not printed_header:
+        printed_header = True
+        print header
+
+    # now we know this is a data line, printing out if the timestamp
+    # has changed, and stacking up otherwise.
+    nextline = process.stdout.readline()
+    date, data = parse_line(nextline)
+    if date != current_ts:
+        current_ts = date
+        print data_line
+        data_line = "%s   %s" % (date, data)
+    else:
+        data_line += "   %s" % data
+
+    sys.stdout.flush()