Merge "parse the failed jobs in stream"

elastic_recheck/elasticRecheck.py

@@ -22,6 +22,7 @@ import urllib2
 import ConfigParser
 import logging
 import os
+import re
 import sys
 import time
 
@@ -57,7 +58,8 @@ class Stream(object):
         if thread:
             self.gerrit.startWatching()
 
-    def _is_jenkins_failure(self, event):
+    @staticmethod
+    def parse_jenkins_failure(event):
         """Is this comment a jenkins failure comment."""
         if event.get('type', '') != 'comment-added':
             return False
@@ -66,8 +68,16 @@ class Stream(object):
         if (username != 'jenkins'):
             return False
 
-        return ("Build failed.  For information on how to proceed" in
-                event['comment'])
+        if not ("Build failed.  For information on how to proceed" in
+                event['comment']):
+            return False
+
+        failed_tests = {}
+        for line in event['comment'].split("\n"):
+            m = re.search("- ([\w-]+)\s*(http://\S+)\s*:\s*FAILURE", line)
+            if m:
+                failed_tests[m.group(1)] = m.group(2)
+        return failed_tests
 
     def _failed_unit_tests(self, line):
         """Did we fail unit tests? If so not a valid failure."""
@@ -84,22 +94,26 @@ class Stream(object):
         self.log.debug("entering get_failed_tempest")
         while True:
             event = self.gerrit.getEvent()
-            if self._is_jenkins_failure(event):
-                self.log.debug("potential failed_tempest")
-                found = False
-                for line in event['comment'].split('\n'):
-                    if self._failed_unit_tests(line):
-                        found = False
-                        break
-                    if self._valid_failure(line):
-                        url = [x for x in line.split() if "http" in x][0]
-                        if RequiredFiles.files_at_url(url):
-                            self.log.debug("All file present")
-                            found = True
-                if found:
-                    return event
+            failed_jobs = Stream.parse_jenkin_failure(event)
+            if not failed_jobs:
+                # nothing to see here, lets try the next event
                 continue
 
+            self.log.debug("potential failed_tempest")
+            found = False
+            for line in event['comment'].split('\n'):
+                if self._failed_unit_tests(line):
+                    found = False
+                    break
+                if self._valid_failure(line):
+                    url = [x for x in line.split() if "http" in x][0]
+                    if RequiredFiles.files_at_url(url):
+                        self.log.debug("All file present")
+                        found = True
+            if found:
+                return event
+            continue
+
     def leave_comment(self, project, commit, bugs=None):
         if bugs:
             bug_urls = ['https://bugs.launchpad.net/bugs/%s' % x for x in bugs]

elastic_recheck/tests/unit/jenkins/events.json

@@ -0,0 +1,20 @@
+{
+    "events": [
+        {
+            "type": "comment-added",
+            "author": {
+                "username": "jenkins"
+            },
+            "comment":"Patch Set 1:\n\nBuild failed.  For information on how to proceed, see https://wiki.openstack.org/wiki/GerritJenkinsGit#Test_Failures\n\n- gate-requirements-pep8 http://logs.openstack.org/31/64831/1/check/gate-requirements-pep8/f5abe44 : SUCCESS in 46s\n- gate-requirements-python27 http://logs.openstack.org/31/64831/1/check/gate-requirements-python27/d09e102 : SUCCESS in 1m 51s\n- gate-requirements-pypy http://logs.openstack.org/31/64831/1/check/gate-requirements-pypy/b5c4672 : SUCCESS in 2m 14s\n- check-requirements-integration-dsvm http://logs.openstack.org/31/64831/1/check/check-requirements-integration-dsvm/135d0b4 : FAILURE in 9m 54s\n- check-tempest-dsvm-full http://logs.openstack.org/31/64831/1/check/check-tempest-dsvm-full/287c655 : FAILURE in 38m 36s\n- check-tempest-dsvm-postgres-full http://logs.openstack.org/31/64831/1/check/check-tempest-dsvm-postgres-full/91f3b16 : FAILURE in 1h 01m 15s\n- check-tempest-dsvm-neutron http://logs.openstack.org/31/64831/1/check/check-tempest-dsvm-neutron/117634b : FAILURE in 31m 20s\n- gate-tempest-dsvm-large-ops http://logs.openstack.org/31/64831/1/check/gate-tempest-dsvm-large-ops/31f47cd : SUCCESS in 14m 28s\n- gate-tempest-dsvm-neutron-large-ops http://logs.openstack.org/31/64831/1/check/gate-tempest-dsvm-neutron-large-ops/ea934a5 : SUCCESS in 15m 42s\n- check-grenade-dsvm http://logs.openstack.org/31/64831/1/check/check-grenade-dsvm/c025bc2 : SUCCESS in 31m 12s\n- check-swift-dsvm-functional http://logs.openstack.org/31/64831/1/check/check-swift-dsvm-functional/91ccf18 : SUCCESS in 12m 14s\n"
+        },
+        {
+            "type": "blah"
+        },
+        {
+            "type": "comment-added",
+            "author": {
+                "username": "sdague"
+            }
+        }
+    ]
+}

elastic_recheck/tests/unit/test_stream.py

@@ -0,0 +1,52 @@
+# Copyright 2014 Samsung Electronics. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from elastic_recheck import elasticRecheck
+from elastic_recheck import tests
+
+
+class TestStream(tests.TestCase):
+
+    def setUp(self):
+        super(TestStream, self).setUp()
+        with open("elastic_recheck/tests/unit/jenkins/events.json") as f:
+            j = json.load(f)
+            self.events = j['events']
+
+    def test_gerrit_parsing_none(self):
+        self.assertFalse(
+            elasticRecheck.Stream.parse_jenkins_failure(self.events[1]))
+        self.assertFalse(
+            elasticRecheck.Stream.parse_jenkins_failure(self.events[2]))
+
+    def test_gerrit_parsing(self):
+        jobs = elasticRecheck.Stream.parse_jenkins_failure(self.events[0])
+        self.assertIn('check-requirements-integration-dsvm', jobs)
+        self.assertIn('check-tempest-dsvm-full', jobs)
+        self.assertIn('check-tempest-dsvm-postgres-full', jobs)
+        self.assertIn('check-tempest-dsvm-neutron', jobs)
+
+        self.assertEqual(jobs['check-requirements-integration-dsvm'],
+                         "http://logs.openstack.org/31/64831/1/check/"
+                         "check-requirements-integration-dsvm/135d0b4")
+
+        self.assertNotIn('gate-requirements-pep8', jobs)
+        self.assertNotIn('gate-requirements-python27', jobs)
+        self.assertNotIn('gate-requirements-pypy', jobs)
+        self.assertNotIn('gate-tempest-dsvm-large-ops', jobs)
+        self.assertNotIn('gate-tempest-dsvm-neutron-large-ops', jobs)
+        self.assertNotIn('check-grenade-dsvm', jobs)
+        self.assertNotIn('check-swift-dsvm-functional', jobs)