diff --git a/elastic_recheck/cmd/graph.py b/elastic_recheck/cmd/graph.py index 22163681..540acac7 100755 --- a/elastic_recheck/cmd/graph.py +++ b/elastic_recheck/cmd/graph.py @@ -21,6 +21,7 @@ import json import os from launchpadlib import launchpad +import pytz import requests import elastic_recheck.elasticRecheck as er @@ -81,9 +82,13 @@ def main(): buglist = [] - epoch = datetime.utcfromtimestamp(0) - ts = datetime.now() - ts = datetime(ts.year, ts.month, ts.day, ts.hour) + # if you don't hate timezones, you don't program enough + epoch = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) + ts = datetime.utcnow().replace(tzinfo=pytz.utc) + # rawnow is useful for sending to javascript + rawnow = int(((ts - epoch).total_seconds()) * 1000) + + ts = datetime(ts.year, ts.month, ts.day, ts.hour).replace(tzinfo=pytz.utc) # ms since epoch now = int(((ts - epoch).total_seconds()) * 1000) # number of days to match to, this should be the same as we are @@ -94,6 +99,18 @@ def main(): # ER timeframe for search timeframe = days * 24 * STEP / 1000 + last_indexed = int( + ((classifier.most_recent() - epoch).total_seconds()) * 1000) + behind = now - last_indexed + + # the data we're going to return, including interesting headers + jsondata = { + 'now': rawnow, + 'last_indexed': last_indexed, + 'behind': behind, + 'buglist': [] + } + for query in classifier.queries: if args.queue: query['query'] = query['query'] + (' AND build_queue:"%s"' % @@ -151,8 +168,9 @@ def main(): buglist = sorted(buglist, key=lambda bug: -(bug['fails24'] * 100000 + bug['fails'])) + jsondata['buglist'] = buglist out = open(args.output, 'w') - out.write(json.dumps(buglist)) + out.write(json.dumps(jsondata)) out.close() diff --git a/elastic_recheck/elasticRecheck.py b/elastic_recheck/elasticRecheck.py index bc28d53d..73990e29 100644 --- a/elastic_recheck/elasticRecheck.py +++ b/elastic_recheck/elasticRecheck.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. - +import dateutil.parser as dp import gerritlib.gerrit import pyelasticsearch @@ -368,6 +368,15 @@ class Classifier(): es_query = qb.generic(query, facet=facet) return self.es.search(es_query, size=size) + def most_recent(self): + """Return the datetime of the most recently indexed event.""" + query = qb.most_recent_event() + results = self.es.search(query, size='1') + if len(results) > 0: + last = dp.parse(results[0].timestamp) + return last + return datetime.datetime.utcfromtimestamp(0) + def classify(self, change_number, patch_number, build_short_uuid, recent=False): """Returns either empty list or list with matched bugs.""" diff --git a/elastic_recheck/query_builder.py b/elastic_recheck/query_builder.py index 3ac023cf..0241f5ec 100644 --- a/elastic_recheck/query_builder.py +++ b/elastic_recheck/query_builder.py @@ -110,3 +110,11 @@ def single_patch(query, review, patch, build_short_uuid): 'AND build_patchset:"%s"' 'AND build_short_uuid:%s' % (query, review, patch, build_short_uuid)) + + +def most_recent_event(): + return generic( + 'filename:console.html ' + 'AND (build_queue:gate OR build_queue:check) ' + 'AND NOT tags:_grokparsefailure ' + 'AND NOT message:"%{logmessage}" ')