From cf0e9d6ef26f39a4afe644aa808c28f7c1c1d708 Mon Sep 17 00:00:00 2001
From: Sean Dague <sean@dague.net>
Date: Fri, 26 Sep 2014 13:18:50 -0400
Subject: [PATCH] add support for computing relevant dates

This records the current time when the data is constructed, the date
of the last valid looking piece of data in elastic search, and how far
behind we seem to be on indexing. The json payload is adjusted to be
able to take additional metadata to support displaying this on the ER
page.

Change-Id: I0068ca0bbe72943d5d92dea704659ed865fea198
---
 elastic_recheck/cmd/graph.py      | 26 ++++++++++++++++++++++----
 elastic_recheck/elasticRecheck.py | 11 ++++++++++-
 elastic_recheck/query_builder.py  |  8 ++++++++
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/elastic_recheck/cmd/graph.py b/elastic_recheck/cmd/graph.py
index 22163681..540acac7 100755
--- a/elastic_recheck/cmd/graph.py
+++ b/elastic_recheck/cmd/graph.py
@@ -21,6 +21,7 @@ import json
 import os
 
 from launchpadlib import launchpad
+import pytz
 import requests
 
 import elastic_recheck.elasticRecheck as er
@@ -81,9 +82,13 @@ def main():
 
     buglist = []
 
-    epoch = datetime.utcfromtimestamp(0)
-    ts = datetime.now()
-    ts = datetime(ts.year, ts.month, ts.day, ts.hour)
+    # if you don't hate timezones, you don't program enough
+    epoch = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
+    ts = datetime.utcnow().replace(tzinfo=pytz.utc)
+    # rawnow is useful for sending to javascript
+    rawnow = int(((ts - epoch).total_seconds()) * 1000)
+
+    ts = datetime(ts.year, ts.month, ts.day, ts.hour).replace(tzinfo=pytz.utc)
     # ms since epoch
     now = int(((ts - epoch).total_seconds()) * 1000)
     # number of days to match to, this should be the same as we are
@@ -94,6 +99,18 @@ def main():
     # ER timeframe for search
     timeframe = days * 24 * STEP / 1000
 
+    last_indexed = int(
+        ((classifier.most_recent() - epoch).total_seconds()) * 1000)
+    behind = now - last_indexed
+
+    # the data we're going to return, including interesting headers
+    jsondata = {
+        'now': rawnow,
+        'last_indexed': last_indexed,
+        'behind': behind,
+        'buglist': []
+    }
+
     for query in classifier.queries:
         if args.queue:
             query['query'] = query['query'] + (' AND build_queue:"%s"' %
@@ -151,8 +168,9 @@ def main():
     buglist = sorted(buglist,
                      key=lambda bug: -(bug['fails24'] * 100000 + bug['fails']))
 
+    jsondata['buglist'] = buglist
     out = open(args.output, 'w')
-    out.write(json.dumps(buglist))
+    out.write(json.dumps(jsondata))
     out.close()
 
 
diff --git a/elastic_recheck/elasticRecheck.py b/elastic_recheck/elasticRecheck.py
index bc28d53d..73990e29 100644
--- a/elastic_recheck/elasticRecheck.py
+++ b/elastic_recheck/elasticRecheck.py
@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
+import dateutil.parser as dp
 import gerritlib.gerrit
 import pyelasticsearch
 
@@ -368,6 +368,15 @@ class Classifier():
             es_query = qb.generic(query, facet=facet)
         return self.es.search(es_query, size=size)
 
+    def most_recent(self):
+        """Return the datetime of the most recently indexed event."""
+        query = qb.most_recent_event()
+        results = self.es.search(query, size='1')
+        if len(results) > 0:
+            last = dp.parse(results[0].timestamp)
+            return last
+        return datetime.datetime.utcfromtimestamp(0)
+
     def classify(self, change_number, patch_number,
                  build_short_uuid, recent=False):
         """Returns either empty list or list with matched bugs."""
diff --git a/elastic_recheck/query_builder.py b/elastic_recheck/query_builder.py
index 3ac023cf..0241f5ec 100644
--- a/elastic_recheck/query_builder.py
+++ b/elastic_recheck/query_builder.py
@@ -110,3 +110,11 @@ def single_patch(query, review, patch, build_short_uuid):
                    'AND build_patchset:"%s"'
                    'AND build_short_uuid:%s' %
                    (query, review, patch, build_short_uuid))
+
+
+def most_recent_event():
+    return generic(
+        'filename:console.html '
+        'AND (build_queue:gate OR build_queue:check) '
+        'AND NOT tags:_grokparsefailure '
+        'AND NOT message:"%{logmessage}" ')