Change in swift-drive-audit handling log rotation.

Change supports kern.log rotation in order to avoid loss
of significant information.

There is a year change functionality added as kern.log
does not keep record of year.

There is also backwards function added which allows
reading logs from the back to the front, speeding up the
execution along with the unit test for it

Fixes Bug 1080682

Change-Id: I93436c405aff5625396514000cab774b66022dd0

AUTHORS

@@ -66,6 +66,7 @@ Paul McMillan (paul.mcmillan@nebula.com)
 Ewan Mellor (ewan.mellor@citrix.com)
 Samuel Merritt (sam@swiftstack.com)
 Stephen Milton (milton@isomedia.com)
+Jola Mirecka (jola.mirecka@hp.com)
 Russ Nelson (russ@crynwr.com)
 Maru Newby (mnewby@internap.com)
 Colin Nicholson (colin.nicholson@iomart.com)

bin/swift-drive-audit

@@ -15,13 +15,14 @@
 # limitations under the License.
 
 import datetime
+import glob
 import os
 import re
 import subprocess
 import sys
 from ConfigParser import ConfigParser
 
-from swift.common.utils import get_logger
+from swift.common.utils import backward, get_logger
 
 
 # To search for more types of errors, add the regex to the list below
@@ -61,27 +62,56 @@ def get_devices(device_dir, logger):
 
 
 def get_errors(minutes):
+    # Assuming log rotation is being used, we need to examine
+    # recently rotated files in case the rotation occured
+    # just before the script is being run - the data we are
+    # looking for may have rotated.
+    log_files = [f for f in glob.glob('/var/log/kern.*[!.][!g][!z]')]
+    log_files.sort()
+
+    now_time = datetime.datetime.now()
+    end_time = now_time - datetime.timedelta(minutes=minutes)
+    # kern.log does not contain the year so we need to keep
+    # track of the year and month in case the year recently
+    # ticked over
+    year = now_time.year
+    prev_entry_month = now_time.month
     errors = {}
-    start_time = datetime.datetime.now() - datetime.timedelta(minutes=minutes)
+
-    try:
+    reached_old_logs = False
-        for line in open('/var/log/kern.log'):
+    for path in log_files:
-            if '[    0.000000]' in line:
+        try:
+            f = open(path)
+        except IOError:
+            logger.error("Error: Unable to open " + path)
+            print("Unable to open " + path)
+            sys.exit(1)
+        for line in backward(f):
+            if '[    0.000000]' in line \
+                or 'KERNEL supported cpus:' in line \
+                    or 'BIOS-provided physical RAM map:' in line:
                 # Ignore anything before the last boot
-                errors = {}
+                reached_old_logs = True
-                continue
+                break
-            log_time_string = '%s %s' % (start_time.year,
+            # Solves the problem with year change - kern.log does not
-                                         ' '.join(line.split()[:3]))
+            # keep track of the year.
+            log_time_entry = line.split()[:3]
+            if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
+                year -= 1
+            prev_entry_month = log_time_entry[0]
+            log_time_string = '%s %s' % (year, ' '.join(log_time_entry))
             log_time = datetime.datetime.strptime(
                 log_time_string, '%Y %b %d %H:%M:%S')
-            if log_time > start_time:
+            if log_time > end_time:
                 for err in error_re:
                     for device in err.findall(line):
                         errors[device] = errors.get(device, 0) + 1
-        return errors
+            else:
-    except IOError:
+                reached_old_logs = True
-        logger.error("Error: Unable to open /var/log/kern.log")
+                break
-        print("Unable to open /var/log/kern.log")
+        if reached_old_logs:
-        sys.exit(1)
+            break
+    return errors
 
 
 def comment_fstab(mount_point):

swift/common/utils.py

@@ -84,6 +84,39 @@ if hash_conf.read('/etc/swift/swift.conf'):
     except (NoSectionError, NoOptionError):
         pass
 
+
+def backward(f, blocksize=4096):
+    """
+    A generator returning lines from a file starting with the last line,
+    then the second last line, etc. i.e., it reads lines backwards.
+    Stops when the first line (if any) is read.
+    This is useful when searching for recent activity in very
+    large files.
+
+    :param f: file object to read
+    :param blocksize: no of characters to go backwards at each block
+    """
+    f.seek(0, os.SEEK_END)
+    if f.tell() == 0:
+        return
+    last_row = ''
+    while f.tell() != 0:
+        try:
+            f.seek(-blocksize, os.SEEK_CUR)
+        except IOError:
+            blocksize = f.tell()
+            f.seek(-blocksize, os.SEEK_CUR)
+        block = f.read(blocksize)
+        f.seek(-blocksize, os.SEEK_CUR)
+        rows = block.split('\n')
+        rows[-1] = rows[-1] + last_row
+        while rows:
+            last_row = rows.pop(-1)
+            if rows and last_row:
+                yield last_row
+    yield last_row
+
+
 # Used when reading config values
 TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
 

test/unit/common/test_utils.py

@@ -147,6 +147,36 @@ class TestUtils(unittest.TestCase):
         self.assertRaises(ValueError, utils.normalize_timestamp, '')
         self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
 
+    def test_backwards(self):
+        """ Test swift.common.utils.backward """
+
+        # The lines are designed so that the function would encounter
+        # all of the boundary conditions and typical conditions.
+        # Block boundaries are marked with '<>' characters
+        blocksize = 25
+        lines = ['123456789x12345678><123456789\n',  # block larger than rest
+                 '123456789x123>\n',  # block ends just before \n character
+                 '123423456789\n',
+                 '123456789x\n',  # block ends at the end of line
+                 '<123456789x123456789x123\n',
+                 '<6789x123\n',  # block ends at the beginning of the line
+                 '6789x1234\n',
+                 '1234><234\n',  # block ends typically in the middle of line
+                 '123456789x123456789\n']
+
+        with TemporaryFile('r+w') as f:
+            for line in lines:
+                f.write(line)
+
+            count = len(lines) - 1
+            for line in utils.backward(f, blocksize):
+                self.assertEquals(line, lines[count].split('\n')[0])
+                count -= 1
+
+        # Empty file case
+        with TemporaryFile('r') as f:
+            self.assertEquals([], list(utils.backward(f)))
+
     def test_mkdirs(self):
         testroot = os.path.join(os.path.dirname(__file__), 'mkdirs')
         try: