Merge "add uncategorized failure generation code"

elastic_recheck/cmd/uncategorized_fails.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+
+# Copyright 2014 Samsung Electronics. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import collections
+import operator
+import re
+
+import jinja2
+
+import elastic_recheck.elasticRecheck as er
+import elastic_recheck.results as er_results
+
+
+def get_options():
+    parser = argparse.ArgumentParser(
+        description='''Build the list of all uncategorized test runs.
+
+        Note: This will take a few minutes to run.''')
+    parser.add_argument('--dir', '-d', help="Queries Directory",
+                        default="queries")
+    parser.add_argument('-t', '--templatedir', help="Template Directory")
+    parser.add_argument('-o', '--output', help="Output File")
+    return parser.parse_args()
+
+
+def setup_template_engine(directory):
+    path = ["web/share/templates"]
+    if directory:
+        path.append(directory)
+
+    loader = jinja2.FileSystemLoader(path)
+    env = jinja2.Environment(loader=loader)
+    return env.get_template("uncategorized.html")
+
+
+def all_fails(classifier):
+    """Find all the the fails in the integrated gate.
+
+    This attempts to find all the build jobs in the integrated gate
+    so we can figure out how good we are doing on total classification.
+    """
+    all_fails = {}
+    query = ('filename:"console.html" '
+             'AND message:"Finished: FAILURE" '
+             'AND build_queue:"gate"')
+    results = classifier.hits_by_query(query, size=30000)
+    facets = er_results.FacetSet()
+    facets.detect_facets(results, ["build_uuid"])
+    for build in facets:
+        for result in facets[build]:
+            # not perfect, but basically an attempt to show the integrated
+            # gate. Would be nice if there was a zuul attr for this in es.
+            if re.search("(^openstack/|devstack|grenade)", result.project):
+                name = result.build_name
+                log = result.log_url.split("console.html")[0]
+                all_fails["%s.%s" % (build, name)] = log
+    return all_fails
+
+
+def num_fails_per_build_name(all_jobs):
+    counts = collections.defaultdict(int)
+    for f in all_jobs:
+        build, job = f.split('.', 1)
+        counts[job] += 1
+    return counts
+
+
+def classifying_rate(fails, data, engine):
+    """Builds and prints the classification rate.
+
+    It's important to know how good a job we are doing, so this
+    tool runs through all the failures we've got and builds the
+    classification rate. For every failure in the gate queue did
+    we find a match for it.
+    """
+    found_fails = {k: False for (k, v) in fails.iteritems()}
+
+    for bugnum in data:
+        bug = data[bugnum]
+        for job in bug['failed_jobs']:
+            found_fails[job] = True
+
+    total = len(found_fails.keys())
+    bad_jobs = collections.defaultdict(int)
+    bad_job_urls = collections.defaultdict(list)
+    count = 0
+    for f in fails:
+        if found_fails[f] is True:
+            count += 1
+        else:
+            build, job = f.split('.', 1)
+            bad_jobs[job] += 1
+            bad_job_urls[job].append(fails[f])
+
+    classifying_rate = ((float(count) / float(total)) * 100.0)
+    sort = sorted(
+        bad_jobs.iteritems(),
+        key=operator.itemgetter(1),
+        reverse=True)
+
+    tvars = {
+        "rate": classifying_rate,
+        "jobs": sort,
+        "urls": bad_job_urls
+    }
+    return engine.render(tvars)
+
+
+def _status_count(results):
+    counts = {}
+    facets = er_results.FacetSet()
+    facets.detect_facets(
+        results,
+        ["build_status", "build_uuid"])
+
+    for key in facets:
+        counts[key] = len(facets[key])
+    return counts
+
+
+def _failure_count(hits):
+    if "FAILURE" in hits:
+        return hits["FAILURE"]
+    else:
+        return 0
+
+
+def _failed_jobs(results):
+    failed_jobs = []
+    facets = er_results.FacetSet()
+    facets.detect_facets(
+        results,
+        ["build_status", "build_uuid"])
+    if "FAILURE" in facets:
+        for build in facets["FAILURE"]:
+            for result in facets["FAILURE"][build]:
+                failed_jobs.append("%s.%s" % (build, result.build_name))
+    return failed_jobs
+
+
+def _count_fails_per_build_name(hits):
+    facets = er_results.FacetSet()
+    counts = collections.defaultdict(int)
+    facets.detect_facets(
+        hits,
+        ["build_status", "build_name", "build_uuid"])
+    if "FAILURE" in facets:
+        for build_name in facets["FAILURE"]:
+            counts[build_name] += 1
+    return counts
+
+
+def _failure_percentage(hits, fails):
+    total_fails_per_build_name = num_fails_per_build_name(fails)
+    fails_per_build_name = _count_fails_per_build_name(hits)
+    per = {}
+    for build in fails_per_build_name:
+        this_job = fails_per_build_name[build]
+        if build in total_fails_per_build_name:
+            total = total_fails_per_build_name[build]
+            per[build] = (float(this_job) / float(total)) * 100.0
+    return per
+
+
+def collect_metrics(classifier, fails):
+    data = {}
+    for q in classifier.queries:
+        results = classifier.hits_by_query(q['query'], size=30000)
+        hits = _status_count(results)
+        data[q['bug']] = {
+            'fails': _failure_count(hits),
+            'hits': hits,
+            'percentages': _failure_percentage(results, fails),
+            'query': q['query'],
+            'failed_jobs': _failed_jobs(results)
+        }
+
+    return data
+
+
+def main():
+    opts = get_options()
+    classifier = er.Classifier(opts.dir)
+    fails = all_fails(classifier)
+    data = collect_metrics(classifier, fails)
+    engine = setup_template_engine(opts.templatedir)
+    html = classifying_rate(fails, data, engine)
+    if opts.output:
+        with open(opts.output, "w") as f:
+            f.write(html)
+    else:
+        print html
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -7,3 +7,4 @@ lockfile
 pbr>=0.5.21,<1.0
 Babel>=0.9.6
 launchpadlib
+Jinja2

setup.cfg

@@ -30,6 +30,7 @@ console_scripts =
     elastic-recheck = elastic_recheck.bot:main
     elastic-recheck-graph = elastic_recheck.cmd.graph:main
     elastic-recheck-success = elastic_recheck.cmd.check_success:main
+    elastic-recheck-uncategorized = elastic_recheck.cmd.uncategorized_fails:main
 
 [build_sphinx]
 source-dir = doc/source

web/share/templates/base.html

@@ -0,0 +1,75 @@
+<html xmlns="http://www.w3.org/1999/xhtml"
+      xmlns:py="http://genshi.edgewall.org/"
+      lang="en">
+  <HEAD>
+    <TITLE>Elastic Recheck</TITLE>
+
+<script type="text/javascript"
+	src="http://status.openstack.org/jquery.min.js"></script>
+<script type="text/javascript"
+	src="http://status.openstack.org/jquery-visibility.min.js"></script>
+<script type="text/javascript"
+	src="http://status.openstack.org/jquery-graphite.js"></script>
+<script type="text/javascript"
+	src="http://status.openstack.org/common.js"></script>
+<script type="text/javascript"
+	src="http://status.openstack.org/jquery.flot.min.js"></script>
+<script type="text/javascript"
+	src="http://status.openstack.org/jquery.flot.time.min.js"></script>
+<script type="text/javascript"
+	src="elastic-recheck.js"></script>
+
+    <!-- Google Fonts -->
+    <link href='http://fonts.googleapis.com/css?family=PT+Sans&amp;subset=latin' rel='stylesheet' type='text/css'/>
+
+    <!-- Framework CSS -->
+    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
+    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
+
+    <!-- IE CSS -->
+    <!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
+
+    <!-- OpenStack Specific CSS -->
+
+    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
+
+    <!-- Page Specific CSS -->
+    <link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
+
+    <link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
+
+
+<style type="text/css">
+.graph {
+    width: 600px;
+    height: 200px;
+    margin-bottom: 6px;
+}
+.extlink {
+    margin-left: 2em;
+    margin-right: 2em;
+}
+.bug-container {
+    margin-bottom: 2em;
+}
+table {
+    width: auto;
+}
+
+</style>
+
+<script type="text/javascript">
+</script>
+
+  </HEAD>
+
+  <BODY>
+    <script type="text/javascript">header('Rechecks');</script>
+    <div id="main-container" class="container">
+    {% block body %}
+    {% endblock %}
+    </div>
+
+    <script type="text/javascript">footer();</script>
+  </BODY>
+</html>

web/share/templates/uncategorized.html

@@ -0,0 +1,33 @@
+{% extends "base.html" %}
+{% block body %}
+    {{ super() }}
+
+<style>
+  .menu {
+     float: right;
+  }
+</style>
+
+<div class="menu">
+<a name="top"></a>
+{% for job in jobs %}
+<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
+{% endfor %}
+</div>
+<div class="jobs">
+<h1>Unclassified failed jobs</h1>
+Overall Categorization Rate: {{ rate }}
+
+
+{% for job in jobs %}
+<a name="{{job[0]}}"></a>
+<a href="#top"><i>back to top</i></a>
+<h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails</h2>
+<ul>
+  {% for url in urls[job[0]] %}
+<li><a href="{{ url }}">{{ url }}</a></li>
+  {% endfor %}
+</ul>
+{% endfor %}
+{% endblock %}
+</div>