Merge "Add crm114 logstash queries to uncategorized page"

This commit is contained in:
Jenkins 2014-09-17 18:30:51 +00:00 committed by Gerrit Code Review
commit f2885a719f
2 changed files with 29 additions and 5 deletions

View File

@ -15,8 +15,10 @@
# under the License. # under the License.
import argparse import argparse
import base64
import collections import collections
import datetime import datetime
import json
import operator import operator
import re import re
@ -72,7 +74,8 @@ def all_fails(classifier):
log = result.log_url.split("console.html")[0] log = result.log_url.split("console.html")[0]
all_fails["%s.%s" % (build, name)] = { all_fails["%s.%s" % (build, name)] = {
'log': log, 'log': log,
'timestamp': timestamp 'timestamp': timestamp,
'build_uuid': result.build_uuid
} }
return all_fails return all_fails
@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
return counts return counts
def classifying_rate(fails, data, engine): def classifying_rate(fails, data, engine, classifier):
"""Builds and prints the classification rate. """Builds and prints the classification rate.
It's important to know how good a job we are doing, so this It's important to know how good a job we are doing, so this
@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
key=lambda v: v['timestamp'], reverse=True) key=lambda v: v['timestamp'], reverse=True)
# Convert timestamp into string # Convert timestamp into string
for url in bad_job_urls[job]: for url in bad_job_urls[job]:
urlq = {}
url['timestamp'] = url['timestamp'].strftime( url['timestamp'] = url['timestamp'].strftime(
"%Y-%m-%dT%H:%M") "%Y-%m-%dT%H:%M")
# setup crm114 query for build_uuid
query = ('build_uuid: "%s" '
'AND error_pr:["-1000.0" TO "-10.0"] '
% url['build_uuid'])
urlq = dict(search=query,
fields=[],
offset=0,
timeframe=str(864000))
logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
results = classifier.hits_by_query(query, size=1)
if results:
url['crm114'] = logstash_url
classifying_rate = collections.defaultdict(int) classifying_rate = collections.defaultdict(int)
classifying_rate['overall'] = "%.1f" % ( classifying_rate['overall'] = "%.1f" % (
@ -231,7 +248,7 @@ def main():
fails = all_fails(classifier) fails = all_fails(classifier)
data = collect_metrics(classifier, fails) data = collect_metrics(classifier, fails)
engine = setup_template_engine(opts.templatedir) engine = setup_template_engine(opts.templatedir)
html = classifying_rate(fails, data, engine) html = classifying_rate(fails, data, engine, classifier)
if opts.output: if opts.output:
with open(opts.output, "w") as f: with open(opts.output, "w") as f:
f.write(html) f.write(html)

View File

@ -62,6 +62,10 @@
<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li> <li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
{% endfor %} {% endfor %}
</div> </div>
<div class='crm114-verbiage'>
The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
</div>
<div class="jobs"> <div class="jobs">
<h1>Unclassified failed jobs</h1> <h1>Unclassified failed jobs</h1>
Overall Categorization Rate: {{ rate['overall'] }}% Overall Categorization Rate: {{ rate['overall'] }}%
@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
<h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2> <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
<ul> <ul>
{% for url in urls[job[0]] %} {% for url in urls[job[0]] %}
{% if url['crm114'] %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
{% else %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li> <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
{% endif %}
{% endfor %} {% endfor %}
</ul> </ul>
{% endfor %} {% endfor %}