opendev/elastic-recheck
Code Issues Proposed changes

time to grow up and fit in 80 columns

Browse Source 
elastic_recheck started off life ignoring the 80 column boundary.
We should stop that, as it's bad form. Also, I do multi column
emacs and it blows my column widths.

So fix all the E501 issues and start enforcing the rules in tox

Change-Id: Ib0a1d48d085d9b21fbc1bab75e93e9cc40d36988
This commit is contained in:
Sean Dague 2013-11-30 07:41:05 -05:00
parent 3d4495d243
commit a176ec483b
7 changed files with 75 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
elastic_recheck/bot.py
View File

@ -185,17 +185,21 @@ def _main(config):
channel_config = ChannelConfig(yaml.load(open(fp))) channel_config = ChannelConfig(yaml.load(open(fp)))
bot = RecheckWatchBot(channel_config.channels, bot = RecheckWatchBot(
config.get('ircbot', 'nick'), channel_config.channels,
config.get('ircbot', 'pass'), config.get('ircbot', 'nick'),
config.get('ircbot', 'server'), config.get('ircbot', 'pass'),
config.getint('ircbot', 'port'), config.get('ircbot', 'server'),
config.get('ircbot', 'server_password')) config.getint('ircbot', 'port'),
recheck = RecheckWatch(bot, channel_config, config.get('ircbot', 'server_password'))
config.get('gerrit', 'user'),
config.get('gerrit', 'query_file'), recheck = RecheckWatch(
config.get('gerrit', 'host', 'review.openstack.org'), bot,
config.get('gerrit', 'key')) channel_config,
config.get('gerrit', 'user'),
config.get('gerrit', 'query_file'),
config.get('gerrit', 'host', 'review.openstack.org'),
config.get('gerrit', 'key'))
recheck.start() recheck.start()
bot.start() bot.start()

3
elastic_recheck/cmd/check_success.py
View File

@ -66,7 +66,8 @@ def print_metrics(data):
sorted_data = sorted(data.iteritems(), sorted_data = sorted(data.iteritems(),
key=lambda x: -x[1]['fails']) key=lambda x: -x[1]['fails'])
for d in sorted_data: for d in sorted_data:
print "Bug: https://bugs.launchpad.net/bugs/%s => %s" % (d[0], d[1]['query'].rstrip()) print("Bug: https://bugs.launchpad.net/bugs/%s => %s"
% (d[0], d[1]['query'].rstrip()))
get_launchpad_bug(d[0]) get_launchpad_bug(d[0])
print "Hits" print "Hits"
for s in d[1]['hits'].keys(): for s in d[1]['hits'].keys():

29
elastic_recheck/elasticRecheck.py
View File

@ -57,6 +57,9 @@ class Stream(object):
if thread: if thread:
self.gerrit.startWatching() self.gerrit.startWatching()
def _is_unit_test(self, line):
return "FAILURE" in line and ("python2" in line or "pep8" in line)
def get_failed_tempest(self): def get_failed_tempest(self):
self.log.debug("entering get_failed_tempest") self.log.debug("entering get_failed_tempest")
while True: while True:
@ -70,7 +73,7 @@ class Stream(object):
self.log.debug("potential failed_tempest") self.log.debug("potential failed_tempest")
found = False found = False
for line in event['comment'].split('\n'): for line in event['comment'].split('\n'):
if "FAILURE" in line and ("python2" in line or "pep8" in line): if self._is_unit_test(line):
# Unit Test Failure # Unit Test Failure
found = False found = False
break break
@ -137,13 +140,16 @@ class Classifier():
#Wait till Elastic search is ready #Wait till Elastic search is ready
self.log.debug("checking if ElasticSearch is ready") self.log.debug("checking if ElasticSearch is ready")
if not self._is_ready(change_number, patch_number, comment): if not self._is_ready(change_number, patch_number, comment):
self.log.error("something went wrong, ElasticSearch is still not ready, " self.log.error(
"giving up and trying next failure") "something went wrong, ElasticSearch is still not ready, "
"giving up and trying next failure")
return None return None
self.log.debug("ElasticSearch is ready, starting to classify") self.log.debug("ElasticSearch is ready, starting to classify")
bug_matches = [] bug_matches = []
for x in self.queries: for x in self.queries:
self.log.debug("Looking for bug: https://bugs.launchpad.net/bugs/%s" % x['bug']) self.log.debug(
"Looking for bug: https://bugs.launchpad.net/bugs/%s"
% x['bug'])
query = qb.single_patch(x['query'], change_number, patch_number) query = qb.single_patch(x['query'], change_number, patch_number)
results = self.es.search(query, size='10') results = self.es.search(query, size='10')
if self._urls_match(comment, results): if self._urls_match(comment, results):
@ -160,7 +166,8 @@ class Classifier():
results = self.es.search(query, size='10') results = self.es.search(query, size='10')
except pyelasticsearch.exceptions.InvalidJsonResponseError: except pyelasticsearch.exceptions.InvalidJsonResponseError:
# If ElasticSearch returns an error code, sleep and retry # If ElasticSearch returns an error code, sleep and retry
#TODO(jogo): if this works pull out search into a helper function that does this. # TODO(jogo): if this works pull out search into a helper
# function that does this.
print "UHUH hit InvalidJsonResponseError" print "UHUH hit InvalidJsonResponseError"
time.sleep(NUMBER_OF_RETRIES) time.sleep(NUMBER_OF_RETRIES)
continue continue
@ -170,7 +177,10 @@ class Classifier():
time.sleep(SLEEP_TIME) time.sleep(SLEEP_TIME)
if i == NUMBER_OF_RETRIES - 1: if i == NUMBER_OF_RETRIES - 1:
return False return False
self.log.debug("Found hits for change_number: %s, patch_number: %s" % (change_number, patch_number)) self.log.debug(
"Found hits for change_number: %s, patch_number: %s"
% (change_number, patch_number))
query = qb.files_ready(change_number, patch_number) query = qb.files_ready(change_number, patch_number)
for i in range(NUMBER_OF_RETRIES): for i in range(NUMBER_OF_RETRIES):
results = self.es.search(query, size='80') results = self.es.search(query, size='80')
@ -182,7 +192,9 @@ class Classifier():
time.sleep(SLEEP_TIME) time.sleep(SLEEP_TIME)
if i == NUMBER_OF_RETRIES - 1: if i == NUMBER_OF_RETRIES - 1:
return False return False
self.log.debug("All files present for change_number: %s, patch_number: %s" % (change_number, patch_number)) self.log.debug(
"All files present for change_number: %s, patch_number: %s"
% (change_number, patch_number))
# Just because one file is parsed doesn't mean all are, so wait a # Just because one file is parsed doesn't mean all are, so wait a
# bit # bit
time.sleep(10) time.sleep(10)
@ -249,7 +261,8 @@ def main():
print "unable to classify failure" print "unable to classify failure"
else: else:
for bug_number in bug_numbers: for bug_number in bug_numbers:
print "Found bug: https://bugs.launchpad.net/bugs/%s" % bug_number print("Found bug: https://bugs.launchpad.net/bugs/%s"
% bug_number)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

16
elastic_recheck/tests/functional/test_gerrit_comment.py
View File

@ -41,7 +41,10 @@ class TestGerritComment(testtools.TestCase):
result = self.gerrit.query(commit, comments=True) result = self.gerrit.query(commit, comments=True)
comments = result.get('comments') comments = result.get('comments')
comment = comments[-1] comment = comments[-1]
self.assertIn("I noticed tempest failed, I think you hit bug https://bugs.launchpad.net/bugs/1223158", comment.get('message')) self.assertIn(
"I noticed tempest failed, I think you hit bug "
"https://bugs.launchpad.net/bugs/1223158",
comment.get('message'))
def test_bugs_found(self): def test_bugs_found(self):
bug_numbers = ['1223158', '424242'] bug_numbers = ['1223158', '424242']
@ -52,7 +55,11 @@ class TestGerritComment(testtools.TestCase):
result = self.gerrit.query(commit, comments=True) result = self.gerrit.query(commit, comments=True)
comments = result.get('comments') comments = result.get('comments')
comment = comments[-1] comment = comments[-1]
self.assertIn("I noticed tempest failed, I think you hit bug https://bugs.launchpad.net/bugs/1223158 and https://bugs.launchpad.net/bugs/424242 and", comment.get('message')) self.assertIn(
"I noticed tempest failed, I think you hit bug "
"https://bugs.launchpad.net/bugs/1223158 and "
"https://bugs.launchpad.net/bugs/424242 and",
comment.get('message'))
def test_bug_not_found(self): def test_bug_not_found(self):
project = 'gtest-org/test' project = 'gtest-org/test'
@ -62,4 +69,7 @@ class TestGerritComment(testtools.TestCase):
result = self.gerrit.query(commit, comments=True) result = self.gerrit.query(commit, comments=True)
comments = result.get('comments') comments = result.get('comments')
comment = comments[-1] comment = comments[-1]
self.assertIn("https://wiki.openstack.org/wiki/GerritJenkinsGithub#Test_Failures", comment.get('message')) self.assertIn(
"https://wiki.openstack.org/wiki/"
"GerritJenkinsGithub#Test_Failures",
comment.get('message'))

10
elastic_recheck/tests/functional/test_queries.py
View File

@ -40,13 +40,16 @@ class TestQueries(tests.TestCase):
def test_queries(self): def test_queries(self):
for x in self.classifier.queries: for x in self.classifier.queries:
print "Looking for bug: https://bugs.launchpad.net/bugs/%s" % x['bug'] print("Looking for bug: https://bugs.launchpad.net/bugs/%s"
% x['bug'])
self.assertTrue((self._is_valid_ElasticSearch_query(x) or self.assertTrue((self._is_valid_ElasticSearch_query(x) or
self._is_valid_launchpad_bug(x['bug'])), self._is_valid_launchpad_bug(x['bug'])),
("Something is wrong with bug %s" % x['bug'])) ("Something is wrong with bug %s" % x['bug']))
def _is_valid_ElasticSearch_query(self, x): def _is_valid_ElasticSearch_query(self, x):
query = self.classifier._apply_template(self.classifier.general_template, x['query']) query = self.classifier._apply_template(
self.classifier.general_template,
x['query'])
results = self.classifier.es.search(query, size='10') results = self.classifier.es.search(query, size='10')
valid_query = int(results['hits']['total']) > 0 valid_query = int(results['hits']['total']) > 0
if not valid_query: if not valid_query:
@ -64,7 +67,8 @@ class TestQueries(tests.TestCase):
bug_tasks = lp_bug.bug_tasks bug_tasks = lp_bug.bug_tasks
bug_complete = map(lambda bug_task: bug_task.is_complete, bug_tasks) bug_complete = map(lambda bug_task: bug_task.is_complete, bug_tasks)
projects = map(lambda bug_task: bug_task.bug_target_name, bug_tasks) projects = map(lambda bug_task: bug_task.bug_target_name, bug_tasks)
# Check if all open bug tasks are closed if is_complete is true for all tasks. # Check if all open bug tasks are closed if is_complete is true
# for all tasks.
if len(bug_complete) != bug_complete.count(True): if len(bug_complete) != bug_complete.count(True):
print "bug %s is closed in launchpad" % bug print "bug %s is closed in launchpad" % bug
return False return False

22
elastic_recheck/tests/unit/test_required_files.py
View File

@ -20,16 +20,22 @@ from elastic_recheck import tests
class TestRequiredFiles(tests.TestCase): class TestRequiredFiles(tests.TestCase):
def test_url(self): def test_url(self):
url = elasticRecheck.RequiredFiles.prep_url('http://logs.openstack.org/13/46613/2/check/gate-tempest-devstack-vm-full/864bf44/console.html') url = elasticRecheck.RequiredFiles.prep_url(
self.assertEqual(url, 'http://logs.openstack.org/13/46613/2/check/'
'http://logs.openstack.org/13/46613/2/check/gate-tempest-devstack-vm-full/864bf44') 'gate-tempest-devstack-vm-full/864bf44/console.html')
self.assertEqual(
url,
'http://logs.openstack.org/13/46613/2/check/'
'gate-tempest-devstack-vm-full/864bf44')
def _fake_urlopen(self, url): def _fake_urlopen(self, url):
pass pass
def test_files_at_url_pass(self): def test_files_at_url_pass(self):
self.stubs.Set(urllib2, 'urlopen', self._fake_urlopen) self.stubs.Set(urllib2, 'urlopen', self._fake_urlopen)
result = elasticRecheck.RequiredFiles.files_at_url('http://logs.openstack.org/13/46613/2/check/gate-tempest-devstack-vm-full/864bf44') result = elasticRecheck.RequiredFiles.files_at_url(
'http://logs.openstack.org/13/46613/2/check/'
'gate-tempest-devstack-vm-full/864bf44')
self.assertTrue(result) self.assertTrue(result)
def _invalid_url_open(self, url): def _invalid_url_open(self, url):
@ -37,5 +43,9 @@ class TestRequiredFiles(tests.TestCase):
def test_files_at_url_fail(self): def test_files_at_url_fail(self):
self.stubs.Set(urllib2, 'urlopen', self._invalid_url_open) self.stubs.Set(urllib2, 'urlopen', self._invalid_url_open)
self.assertFalse(elasticRecheck.RequiredFiles.files_at_url('http://logs.openstack.org/02/44502/7/check/gate-tempest-devstack-vm-neutron/4f386e5')) self.assertFalse(elasticRecheck.RequiredFiles.files_at_url(
self.assertFalse(elasticRecheck.RequiredFiles.files_at_url('http://logs.openstack.org/45/47445/3/check/gate-tempest-devstack-vm-full/0e43e09/')) 'http://logs.openstack.org/02/44502/7/check/'
'gate-tempest-devstack-vm-neutron/4f386e5'))
self.assertFalse(elasticRecheck.RequiredFiles.files_at_url(
'http://logs.openstack.org/45/47445/3/check/'
'gate-tempest-devstack-vm-full/0e43e09/'))

3
tox.ini
View File

@ -29,7 +29,6 @@ commands = python setup.py testr --coverage --testr-args='{posargs}'
[flake8] [flake8]
# H803 Skipped on purpose # H803 Skipped on purpose
# E501 skipped temporarily
ignore = E123,E122,E126,E128,E501,H803 ignore = E123,E122,E126,E128,H803
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build