openstack/swift
Code Issues Proposed changes

Change in swift-drive-audit handling log rotation.

Browse Source 
Change supports kern.log rotation in order to avoid loss
of significant information.

There is a year change functionality added as kern.log
does not keep record of year.

There is also backwards function added which allows
reading logs from the back to the front, speeding up the
execution along with the unit test for it

Fixes Bug 1080682

Change-Id: I93436c405aff5625396514000cab774b66022dd0
This commit is contained in:
jola-mirecka 2012-11-19 13:02:34 +00:00
parent af3bd46679
commit 902b66d3ae
4 changed files with 109 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -66,6 +66,7 @@ Paul McMillan (paul.mcmillan@nebula.com)
Ewan Mellor (ewan.mellor@citrix.com)
Samuel Merritt (sam@swiftstack.com)
Stephen Milton (milton@isomedia.com)
Jola Mirecka (jola.mirecka@hp.com)
Russ Nelson (russ@crynwr.com)
Maru Newby (mnewby@internap.com)
Colin Nicholson (colin.nicholson@iomart.com)

60
bin/swift-drive-audit
View File

@ -15,13 +15,14 @@
# limitations under the License.
import datetime
import glob
import os
import re
import subprocess
import sys
from ConfigParser import ConfigParser
from swift.common.utils import get_logger
from swift.common.utils import backward, get_logger
# To search for more types of errors, add the regex to the list below
@ -61,27 +62,56 @@ def get_devices(device_dir, logger):
def get_errors(minutes):
# Assuming log rotation is being used, we need to examine
# recently rotated files in case the rotation occured
# just before the script is being run - the data we are
# looking for may have rotated.
log_files = [f for f in glob.glob('/var/log/kern.*[!.][!g][!z]')]
log_files.sort()
now_time = datetime.datetime.now()
end_time = now_time - datetime.timedelta(minutes=minutes)
# kern.log does not contain the year so we need to keep
# track of the year and month in case the year recently
# ticked over
year = now_time.year
prev_entry_month = now_time.month
errors = {}
start_time = datetime.datetime.now() - datetime.timedelta(minutes=minutes)
try:
for line in open('/var/log/kern.log'):
if '[ 0.000000]' in line:
reached_old_logs = False
for path in log_files:
try:
f = open(path)
except IOError:
logger.error("Error: Unable to open " + path)
print("Unable to open " + path)
sys.exit(1)
for line in backward(f):
if '[ 0.000000]' in line \
or 'KERNEL supported cpus:' in line \
or 'BIOS-provided physical RAM map:' in line:
# Ignore anything before the last boot
errors = {}
continue
log_time_string = '%s %s' % (start_time.year,
' '.join(line.split()[:3]))
reached_old_logs = True
break
# Solves the problem with year change - kern.log does not
# keep track of the year.
log_time_entry = line.split()[:3]
if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
year -= 1
prev_entry_month = log_time_entry[0]
log_time_string = '%s %s' % (year, ' '.join(log_time_entry))
log_time = datetime.datetime.strptime(
log_time_string, '%Y %b %d %H:%M:%S')
if log_time > start_time:
if log_time > end_time:
for err in error_re:
for device in err.findall(line):
errors[device] = errors.get(device, 0) + 1
return errors
except IOError:
logger.error("Error: Unable to open /var/log/kern.log")
print("Unable to open /var/log/kern.log")
sys.exit(1)
else:
reached_old_logs = True
break
if reached_old_logs:
break
return errors
def comment_fstab(mount_point):

33
swift/common/utils.py
View File

@ -84,6 +84,39 @@ if hash_conf.read('/etc/swift/swift.conf'):
except (NoSectionError, NoOptionError):
pass
def backward(f, blocksize=4096):
"""
A generator returning lines from a file starting with the last line,
then the second last line, etc. i.e., it reads lines backwards.
Stops when the first line (if any) is read.
This is useful when searching for recent activity in very
large files.
:param f: file object to read
:param blocksize: no of characters to go backwards at each block
"""
f.seek(0, os.SEEK_END)
if f.tell() == 0:
return
last_row = ''
while f.tell() != 0:
try:
f.seek(-blocksize, os.SEEK_CUR)
except IOError:
blocksize = f.tell()
f.seek(-blocksize, os.SEEK_CUR)
block = f.read(blocksize)
f.seek(-blocksize, os.SEEK_CUR)
rows = block.split('\n')
rows[-1] = rows[-1] + last_row
while rows:
last_row = rows.pop(-1)
if rows and last_row:
yield last_row
yield last_row
# Used when reading config values
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))

30
test/unit/common/test_utils.py
View File

@ -147,6 +147,36 @@ class TestUtils(unittest.TestCase):
self.assertRaises(ValueError, utils.normalize_timestamp, '')
self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
def test_backwards(self):
""" Test swift.common.utils.backward """
# The lines are designed so that the function would encounter
# all of the boundary conditions and typical conditions.
# Block boundaries are marked with '<>' characters
blocksize = 25
lines = ['123456789x12345678><123456789\n', # block larger than rest
'123456789x123>\n', # block ends just before \n character
'123423456789\n',
'123456789x\n', # block ends at the end of line
'<123456789x123456789x123\n',
'<6789x123\n', # block ends at the beginning of the line
'6789x1234\n',
'1234><234\n', # block ends typically in the middle of line
'123456789x123456789\n']
with TemporaryFile('r+w') as f:
for line in lines:
f.write(line)
count = len(lines) - 1
for line in utils.backward(f, blocksize):
self.assertEquals(line, lines[count].split('\n')[0])
count -= 1
# Empty file case
with TemporaryFile('r') as f:
self.assertEquals([], list(utils.backward(f)))
def test_mkdirs(self):
testroot = os.path.join(os.path.dirname(__file__), 'mkdirs')
try: