Merge "Add crm114 logstash queries to uncategorized page"

elastic_recheck/cmd/uncategorized_fails.py

@@ -15,8 +15,10 @@
 #    under the License.
 
 import argparse
+import base64
 import collections
 import datetime
+import json
 import operator
 import re
 
@@ -72,7 +74,8 @@ def all_fails(classifier):
                 log = result.log_url.split("console.html")[0]
                 all_fails["%s.%s" % (build, name)] = {
                     'log': log,
-                    'timestamp': timestamp
+                    'timestamp': timestamp,
+                    'build_uuid': result.build_uuid
                 }
     return all_fails
 
@@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
     return counts
 
 
-def classifying_rate(fails, data, engine):
+def classifying_rate(fails, data, engine, classifier):
     """Builds and prints the classification rate.
 
     It's important to know how good a job we are doing, so this
@@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
                                    key=lambda v: v['timestamp'], reverse=True)
         # Convert timestamp into string
         for url in bad_job_urls[job]:
+            urlq = {}
             url['timestamp'] = url['timestamp'].strftime(
                 "%Y-%m-%dT%H:%M")
+            # setup crm114 query for build_uuid
+            query = ('build_uuid: "%s" '
+                     'AND error_pr:["-1000.0" TO "-10.0"] '
+                     % url['build_uuid'])
+            urlq = dict(search=query,
+                        fields=[],
+                        offset=0,
+                        timeframe=str(864000))
+            logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
+            logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
+            results = classifier.hits_by_query(query, size=1)
+            if results:
+                url['crm114'] = logstash_url
 
     classifying_rate = collections.defaultdict(int)
     classifying_rate['overall'] = "%.1f" % (
@@ -231,7 +248,7 @@ def main():
     fails = all_fails(classifier)
     data = collect_metrics(classifier, fails)
     engine = setup_template_engine(opts.templatedir)
-    html = classifying_rate(fails, data, engine)
+    html = classifying_rate(fails, data, engine, classifier)
     if opts.output:
         with open(opts.output, "w") as f:
             f.write(html)

web/share/templates/uncategorized.html

@@ -62,6 +62,10 @@
 <li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
 {% endfor %}
 </div>
+<div class='crm114-verbiage'>
+The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
+More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
+</div>
 <div class="jobs">
 <h1>Unclassified failed jobs</h1>
 Overall Categorization Rate: {{ rate['overall'] }}%
@@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
 <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
 <ul>
   {% for url in urls[job[0]] %}
-
+    {% if url['crm114'] %}
+      <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
+    {% else %}
       <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+    {% endif %}
   {% endfor %}
 </ul>
 {% endfor %}