openstack/swift
Code Issues Proposed changes

added (working) stats tests

Browse Source
This commit is contained in:
John Dickinson 2010-09-01 17:06:01 -05:00
parent fc9e26a44e
commit 7763e541eb
2 changed files with 16 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
swift/stats/stats_processor.py
View File

@ -21,7 +21,6 @@ class StatsLogProcessor(object):
def process(self, obj_stream): def process(self, obj_stream):
'''generate hourly groupings of data from one stats log file''' '''generate hourly groupings of data from one stats log file'''
account_totals = {} account_totals = {}
year, month, day, hour, _ = item.split('/')
for line in obj_stream: for line in obj_stream:
if not line: if not line:
continue continue
@ -32,13 +31,11 @@ class StatsLogProcessor(object):
bytes_used, bytes_used,
created_at) = line.split(',') created_at) = line.split(',')
account = account.strip('"') account = account.strip('"')
if account_name and account_name != account:
continue
container_count = int(container_count.strip('"')) container_count = int(container_count.strip('"'))
object_count = int(object_count.strip('"')) object_count = int(object_count.strip('"'))
bytes_used = int(bytes_used.strip('"')) bytes_used = int(bytes_used.strip('"'))
aggr_key = (account, year, month, day, hour) created_at = created_at.strip('"')
d = account_totals.get(aggr_key, {}) d = account_totals.get(account, {})
d['count'] = d.setdefault('count', 0) + 1 d['count'] = d.setdefault('count', 0) + 1
d['container_count'] = d.setdefault('container_count', 0) + \ d['container_count'] = d.setdefault('container_count', 0) + \
container_count container_count
@ -47,8 +44,8 @@ class StatsLogProcessor(object):
d['bytes_used'] = d.setdefault('bytes_used', 0) + \ d['bytes_used'] = d.setdefault('bytes_used', 0) + \
bytes_used bytes_used
d['created_at'] = created_at d['created_at'] = created_at
account_totals[aggr_key] = d account_totals[account] = d
except (IndexError, ValueError): except (IndexError, ValueError):
# bad line data # bad line data
pass pass
return account_totals, item return account_totals

55
test/unit/stats/test_log_processor.py
View File

@ -38,7 +38,7 @@ class TestLogProcessor(unittest.TestCase):
'/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\ '/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\
'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\ 'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\
'6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262' '6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262'
stats_test_line = 'account,1,2,3' stats_test_line = 'account,1,2,3,1283378584.881391'
proxy_config = {'log-processor': { proxy_config = {'log-processor': {
} }
@ -106,21 +106,6 @@ class TestLogProcessor(unittest.TestCase):
'prefix_query': 0}} 'prefix_query': 0}}
self.assertEquals(result, expected) self.assertEquals(result, expected)
def test_process_one_stats_file(self):
return
p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
def get_object_data(*a,**kw):
return [self.stats_test_line]
p.get_object_data = get_object_data
result = p.process_one_stats_file('y/m/d/h/f', None)
expected = ({('account', 'y', 'm', 'd', 'h'):
{'count': 1,
'object_count': 2,
'container_count': 1,
'bytes_used': 3}},
'y/m/d/h/f')
self.assertEquals(result, expected)
def test_get_container_listing(self): def test_get_container_listing(self):
p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy() p.internal_proxy = DumbInternalProxy()
@ -152,38 +137,22 @@ class TestLogProcessor(unittest.TestCase):
self.assertEquals(result, expected) self.assertEquals(result, expected)
def test_get_stat_totals(self): def test_get_stat_totals(self):
return stats_proxy_config = self.proxy_config
p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) stats_proxy_config.update({
'log-processor-stats': {
'class_path':
'swift.stats.stats_processor.StatsLogProcessor'
}})
p = log_processor.LogProcessor(stats_proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy() p.internal_proxy = DumbInternalProxy()
def get_object_data(*a,**kw): def get_object_data(*a,**kw):
return [self.stats_test_line] return [self.stats_test_line]
p.get_object_data = get_object_data p.get_object_data = get_object_data
result = list(p.get_stat_totals()) result = p.process_one_file('stats', 'a', 'c', 'o')
expected = [({('account', '2010', '03', '14', '13'): expected = {'account':
{'count': 1, {'count': 1,
'object_count': 2, 'object_count': 2,
'container_count': 1, 'container_count': 1,
'bytes_used': 3}}, 'bytes_used': 3,
'2010/03/14/13/obj1')] 'created_at': '1283378584.881391'}}
self.assertEquals(result, expected)
def test_get_aggr_access_logs(self):
return
p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy()
def get_object_data(*a,**kw):
return [self.access_test_line]
p.get_object_data = get_object_data
result = list(p.get_aggr_access_logs())
expected = [({('AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31', '2010', '07', '09', '04'):
{('public', 'object', 'GET', '2xx'): 1,
('public', 'bytes_out'): 95,
'marker_query': 0,
'format_query': 1,
'delimiter_query': 0,
'path_query': 0,
('public', 'bytes_in'): 0,
'prefix_query': 0}},
'2010/03/14/13/obj1',
{})]
self.assertEquals(result, expected) self.assertEquals(result, expected)