Refactor candidacy checking tools

This is a large change as it does several things

1. Moves to using the gerrit REST API to collecting open reviews (as
   opposed to ssh)
2. Moves the selection of validating commits from scraping git logs to
   the gerrit REST API
3. Uses the new utils.py function for getting the project data
4. Avoids any git checkouts/downloads during the validation process
5. tools/check-new-candidacy still checks all open changes
6. tools/check-ptl-candidacy.py checks a single change, now specified as
   a change ID rather then a file path

Work to be done in follow-up commits

1. Add testing, now that this is more modular we can mock json blobs to
   test code and avoid regressions
2. Add a manual toll to specify a project and community member and check
   that, even if it doesn't match data in an open review
3. Add quality tools such as flake8 / yamllint to keep code/data neat
4. Modify the docs generation to understand that files are now IRC nicks
   and to get the change author as the candidate.

Change-Id: Ibd7fad3eb4d39f1edca624b981fa602d2b4c4d40

requirements.txt

@@ -3,3 +3,5 @@
 # process, which may cause wedges in the gate later.
 pytz>=2013.6 # MIT
 PyYAML>=3.1.0 # MIT
+requests>=2.10.0 # Apache-2.0
+ndg-httpsclient>=0.4.2;python_version<'3.0' # BSD

tools/check-candidacy.py

@@ -1,104 +1,59 @@
 #!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
 
-import yaml
+from __future__ import absolute_import
-import os
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
 import sys
-import urllib
-import re
-import datetime
-import pytz
 
-DATE_MIN = '2015-03-04'
+import check_candidacy
-DATE_MAX = '2016-03-03'
+import utils
-
-BASE_URL = 'https://git.openstack.org/cgit'
-PROJECTS_TAG = 'march-2016-elections'
-PROJECTS_URL = ('%s/openstack/governance/plain/reference/projects.yaml?%s' %
-                (BASE_URL, PROJECTS_TAG))
-
-date_min = datetime.datetime.strptime(DATE_MIN, '%Y-%m-%d').strftime('%s')
-date_max = datetime.datetime.strptime(DATE_MAX, '%Y-%m-%d').strftime('%s')
-now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc,
-                                         hour=0,
-                                         minute=0,
-                                         second=0,
-                                         microsecond=0)
 
 
-def check_atc_date(atc):
+def main():
-    if 'expires-in' not in atc:
+    description = ('Check if the owner of a change is a valid candidate as '
-        return False
+                   'described in the change')
-    expires_in = datetime.datetime.strptime(atc['expires-in'], '%B %Y')
+    parser = argparse.ArgumentParser(description)
-    expires_in = expires_in.replace(tzinfo=pytz.utc)
+    parser.add_argument(dest='change_id', help=('A valid gerrit change ID'))
-    return now < expires_in
+    parser.add_argument('--limit', dest='limit', type=int, default=1,
+                        help=('How many validating changes to report.  '
+                              'A negative value means report many.  '
+                              'Default: %(default)s'))
+    parser.add_argument('--tag', dest='tag', default=utils.PROJECTS_TAG,
+                        help=('The governance tag to validate against.  '
+                              'Default: %(default)s'))
 
-
+    args = parser.parse_args()
-def check_date(date):
+    review = utils.get_reviews(args.change_id)[0]
-    epoch = datetime.datetime.strptime(date, '%Y-%m-%d').strftime('%s')
+    owner = review.get('owner', {})
-    if epoch > date_min and epoch < date_max:
+    if args.limit < 0:
-        return True
+        args.limit = 100
-    return False
-
-
-def escape_author(author):
-    author = author.replace(' ', '+')
-    author = author.replace('_', '+')
-
-    return author
 
     try:
-    project_name = os.path.basename(os.path.dirname(sys.argv[1]))
+        found = check_candidacy.check_candidacy(review['change_id'],
-    author = os.path.basename(sys.argv[1])[:-4]
+                                                review=review)
-except:
+    except Exception as exc:
-    print "usage: %s candidacy_file" % sys.argv[0]
+        print("[E] %s\n\n" % (exc))
-    exit(1)
-
-author = author.replace('_', ' ')
-
-if not os.path.isfile('.projects.yaml'):
-    open('.projects.yaml', 'w').write(
-        urllib.urlopen(PROJECTS_URL).read()
-    )
-projects = yaml.load(open('.projects.yaml'))
-project_list = None
-
-if project_name == "TC":
-    project_list = projects.values()
     else:
-    for key in projects.keys():
-        if key.title().replace(' ', '_') == project_name:
-            project_list = [projects[key]]
-            break
-
-if project_list is None:
-    print "Can't find project [%s] in %s" % (project_name, projects.keys())
-    exit(1)
-
-for project in project_list:
-    if 'extra-atcs' in project:
-        for atc in project['extra-atcs']:
-            if atc['name'] == author and check_atc_date(atc):
-                print "Valid extra ATC record", atc
-                exit(0)
-    for deliverable in project['deliverables'].values():
-        for repo_name in deliverable["repos"]:
-            url = '%s/%s/log/?qt=author&q=%s' % (BASE_URL, repo_name,
-                                                 escape_author(author))
-            print "Querying: %s" % url
-            found = False
-            for l in urllib.urlopen(url).read().split('\n'):
-                if "commit/?id=" not in l:
-                    continue
-                try:
-                    url = ('http://git.openstack.org%s' %
-                           re.search("href='([^']*)'", l).groups()[0])
-                    date = re.search('<td>([^<]*)</td>', l).groups()[0]
-                    if not check_date(date):
-                        continue
-                except:
-                    continue
-                print "[%s]: %s" % (date, url)
-                found = True
         if found:
-                exit(0)
+            print('SUCESS: %s is a valid candidate\n\n' % (owner['email']))
-exit(1)
+            return 0
+        else:
+            print('[E]: %s is not a valid candidate\n\n' % (owner['email']))
+            return 1
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/check-new-candidacy.py

@@ -1,46 +1,59 @@
 #!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
 
-import subprocess
+from __future__ import absolute_import
-import json
+from __future__ import print_function
+from __future__ import unicode_literals
 
+import sys
 
-def pread(argv):
+import check_candidacy
-    return subprocess.Popen(argv, stdout=subprocess.PIPE).communicate()[0]
+import utils
-
-
-def execute(argv):
-    return subprocess.Popen(argv).wait()
 
 
 def get_reviews():
-    reviews = pread(["ssh", "-p", "29418", "review.openstack.org", "gerrit",
+    return utils.get_reviews('is:open project:%s' %
-                     "query", "--format=JSON", "status:open",
+                             (utils.ELECTION_REPO))
-                     "project:'openstack/election'"]).split('\n')
-    results = []
-    for i in reviews:
-        if "status" not in i:
-            continue
-        review = json.loads(i)
-        if review['status'] == 'NEW':
-            results.append(int(review['number']))
-    return results
 
 
 def check_reviews():
     for review in get_reviews():
-        if execute(["git", "review", "-d", "%d" % review]):
+        if review['status'] != 'NEW':
             continue
-        print
+
-        fl = filter(lambda x: x.startswith("candidates/"),
+        print('Checking %s/%d' %
-                    pread(["git", "diff", "--name-only", "HEAD^"]).split('\n'))
+              (utils.GERRIT_BASE, review['_number']))
-        if not len(fl):
+
-            print "[E] No candidacy added"
+        if not len(utils.candidate_files(review)):
+            print("[E] No candidacy added")
             continue
-        for candidate in fl:
+
-            print "[->] https://review.openstack.org/%d  - %s" % (
+        owner = review.get('owner', {})
-                review, candidate)
+        try:
-            execute(["./tools/check-candidacy.py", candidate])
+            found = check_candidacy.check_candidacy(review['change_id'],
-    execute(["git", "checkout", "master"])
+                                                    review=review)
+        except Exception as exc:
+            print("[E] %s\n\n" % (exc))
+        else:
+            if found:
+                print('SUCESS: %s is a valid candidate\n\n' %
+                      (owner['email']))
+            else:
+                print('[E]: %s is not a valid candidate\n\n' %
+                      (owner['email']))
+    return 0
+
 
 if __name__ == "__main__":
-    check_reviews()
+    sys.exit(check_reviews())

tools/check_candidacy.py

@@ -0,0 +1,78 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import os
+
+import utils
+
+
+# FIXME: Printing from library function isn't great.
+#        change API to return the messages and let the consumer decide what to
+#        do with them
+def check_candidacy(change_id, limit=1, tag=None, review=None):
+    def pretty_datetime(dt_str):
+        dt = datetime.datetime.strptime(dt_str.split('.')[0],
+                                        '%Y-%m-%d %H:%M:%S')
+        return dt.strftime('%Y-%m-%d')
+
+    projects = utils.get_projects(tag=tag)
+
+    # If there is more than one review that matches this change_id then all
+    # bets are off
+    review = review or utils.get_reviews(change_id)[0]
+    owner = review.get('owner', {})
+    found = 0
+
+    for filename in utils.candidate_files(review):
+        _, series, project_name, candidate_file = filename.split(os.sep)
+
+        if project_name != 'TC':
+            project_name = utils.dir2name(project_name, projects)
+
+        if project_name in ['Stable branch Maintenance']:
+            project_list = projects.values()
+        else:
+            project_list = [projects[project_name]]
+
+        for project in project_list:
+            for atc in project.get('extra-atcs', []):
+                if (atc['email'] == owner['email'] and
+                        utils.check_atc_date(atc)):
+                    print("Valid extra ATC record:\n\t%s" % (atc))
+                    found += 1
+                    if found >= limit:
+                        return True
+
+            for deliverable in project['deliverables'].values():
+                for repo_name in deliverable["repos"]:
+                    query = ('is:merged after:"%s" before:"%s" '
+                             'owner:%s project:%s' %
+                             (utils.gerrit_datetime(utils.PERIOD_START),
+                              utils.gerrit_datetime(utils.PERIOD_END),
+                              owner['email'], repo_name))
+                    print('Checking %s for merged changes by %s' %
+                          (repo_name, owner['email']))
+                    for review in utils.get_reviews(query):
+                        url = ('%s/#/q/%s' %
+                               (utils.GERRIT_BASE, review['change_id']))
+                        print('%2d: %s %s' %
+                              (found, pretty_datetime(review['submitted']),
+                               url))
+                        found += 1
+                        if found >= limit:
+                            return True
+    return found > 0

tools/utils.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -12,17 +10,73 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import json
 import os
+import pickle
+import pytz
+import requests
 import time
 import urllib
 import yaml
-import pickle
 
+# Per election constants
+
+SERIES_NAME = 'ocata'
+# 2015-09-05 00:00:00 +0000
+PERIOD_START = datetime.datetime(2015, 9, 5, 0, 0, 0, tzinfo=pytz.utc)
+# 2016-09-04 23:59:59 +0000
+PERIOD_END = datetime.datetime(2016, 9, 4, 23, 59, 59, tzinfo=pytz.utc)
+PROJECTS_TAG = 'sept-2016-elections'
+
+# Library constants
+GERRIT_BASE = 'https://review.openstack.org'
+ELECTION_REPO = 'openstack/election'
 BASE_URL = 'https://git.openstack.org/cgit'
 PROJECTS_URL = ('%s/openstack/governance/plain/reference/projects.yaml' %
                 (BASE_URL))
 
 
+# Gerrit functions
+def gerrit_datetime(dt):
+    return dt.strftime('%Y-%m-%d %H:%M:%S %z')
+
+
+def gerrit_query(url):
+    r = requests.get(url)
+    if r.status_code == 200:
+        data = json.loads(r.text[4:])
+    else:
+        data = []
+    return data
+
+
+def get_reviews(query):
+    opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS']
+    opts_str = '&o=%s' % ('&o='.join(opts))
+    url = ('%s/changes/?q=%s%s' %
+           (GERRIT_BASE, urllib.quote_plus(query, safe='/:=><'), opts_str))
+    return gerrit_query(url)
+
+
+def candidate_files(review):
+    return list(filter(lambda x: x.startswith("candidates/"),
+                       list(review['revisions'].values())[0]['files'].keys()))
+
+
+# Governance functions
+def check_atc_date(atc):
+    if 'expires-in' not in atc:
+        return False
+    expires_in = datetime.datetime.strptime(atc['expires-in'], '%B %Y')
+    expires_in = expires_in.replace(tzinfo=pytz.utc)
+    return PERIOD_END < expires_in
+
+
 def get_projects(tag=None):
     url = PROJECTS_URL
     cache_file = '.projects.pkl'
@@ -35,13 +89,14 @@ def get_projects(tag=None):
     if (not os.path.isfile(cache_file) or
             os.stat(cache_file).st_size < 100 or
             os.stat(cache_file).st_mtime + (7*24*3600) < time.time()):
-        print "[+] Updating %s" % cache_file
+        print("[+] Updating %s" % (cache_file))
         data = yaml.safe_load(urllib.urlopen(url).read())
         pickle.dump(data, open(cache_file, "w"))
 
     return pickle.load(open(cache_file))
 
 
+# Election functions
 def name2dir(name):
     """Convert project name to directory name: only [a-zA-Z_] in camelcase"""
     name = name.replace(' ', '_').replace('-', '_')
@@ -55,3 +110,4 @@ def dir2name(name, projects):
         pname = project_name.lower().replace(' ', '').replace('-', '').lower()
         if name == pname:
             return project_name
+    raise ValueError(('%s does not match any project' % (name)))