opendev/elastic-recheck
Code Issues Proposed changes

Add crm114 logstash queries to uncategorized page

Browse Source 
Adding code to add in links to crm114 logstash queries
for those entries on the uncategorized page.

We only add a link if results were found during page
production.  This is to not waste human time on empty
queries.  However, this is a nice-to-have and can
be removed should any objection exist.

We also add some verbiage at the top of the uncategorized
page that describes what the crm114 links are as well
as including links to more in-depth docs, which are added
with change I086b2dfa68ca2239d70cec6db0b22951e6786ed7.

Change-Id: Ie572ef90a3393b43d28608b95a8a9b24bd014962
This commit is contained in:
pcrews 2014-08-12 22:00:02 -07:00
parent f3ef417122
commit c7b42bb88d
2 changed files with 29 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
elastic_recheck/cmd/uncategorized_fails.py
View File

@ -15,8 +15,10 @@
# under the License.
import argparse
import base64
import collections
import datetime
import json
import operator
import re
@ -72,7 +74,8 @@ def all_fails(classifier):
log = result.log_url.split("console.html")[0]
all_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp
'timestamp': timestamp,
'build_uuid': result.build_uuid
}
return all_fails
@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
return counts
def classifying_rate(fails, data, engine):
def classifying_rate(fails, data, engine, classifier):
"""Builds and prints the classification rate.
It's important to know how good a job we are doing, so this
@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
key=lambda v: v['timestamp'], reverse=True)
# Convert timestamp into string
for url in bad_job_urls[job]:
urlq = {}
url['timestamp'] = url['timestamp'].strftime(
"%Y-%m-%dT%H:%M")
# setup crm114 query for build_uuid
query = ('build_uuid: "%s" '
'AND error_pr:["-1000.0" TO "-10.0"] '
% url['build_uuid'])
urlq = dict(search=query,
fields=[],
offset=0,
timeframe=str(864000))
logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
results = classifier.hits_by_query(query, size=1)
if results:
url['crm114'] = logstash_url
classifying_rate = collections.defaultdict(int)
classifying_rate['overall'] = "%.1f" % (
@ -231,7 +248,7 @@ def main():
fails = all_fails(classifier)
data = collect_metrics(classifier, fails)
engine = setup_template_engine(opts.templatedir)
html = classifying_rate(fails, data, engine)
html = classifying_rate(fails, data, engine, classifier)
if opts.output:
with open(opts.output, "w") as f:
f.write(html)

11
web/share/templates/uncategorized.html
View File

@ -62,6 +62,10 @@
<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
{% endfor %}
</div>
<div class='crm114-verbiage'>
The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
</div>
<div class="jobs">
<h1>Unclassified failed jobs</h1>
Overall Categorization Rate: {{ rate['overall'] }}%
@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
<h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
<ul>
{% for url in urls[job[0]] %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
{% if url['crm114'] %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
{% else %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
{% endif %}
{% endfor %}
</ul>
{% endfor %}