Support for running tools under python3

If we run the tools (create-directories or check-all-candidacies) under
python3 they'll fail with silly errors (like[1]).  This is because we
have a number of assumptions they we'll be running under python2 like:

 - pickle.protocol: we done specify one so if we write a pickle file
   from python3 we can't read it in python2.
 - urllib and urlparse: those modules change paths in python 2 or 3 so
   use the six helpers to get the bits we want.
 - Explicitly write files in binary format
 - Various str vs bytes issues
 - dict().keys() returns dict_keys under python3 explictly cast that to
   list()

After addressing these issues we can run the tools under either python2
or python3.  To make this easy to check create explicit ven2 and venv3
test environments, leaving all other testenvs to use the system default.

[1] http://paste.openstack.org/show/593823/
Change-Id: I20334a52500847c810b486c9c8b108e75a5d6303

doc/source/_exts/candidates.py

@@ -33,7 +33,7 @@ def render_list(list_type, candidates_list):
     output_file = os.path.join(".", "doc", "source", "%s.rst" % list_type)
     template_name = "%s.jinja" % list_type
     template_dir = os.path.join(".", "doc", "source", "_exts")
-    with open(output_file, "w") as out:
+    with open(output_file, "wb") as out:
         out.write(
             render_template(
                 template_name,
@@ -59,7 +59,7 @@ def build_archive(serie, list_type):
     output = os.path.join(".", "doc", "source", serie, "%s.rst" % list_type)
     template_name = "%s_archive.jinja" % list_type
     template_dir = os.path.join(".", "doc", "source", "_exts")
-    with open(output, "w") as out:
+    with open(output, "wb") as out:
         out.write(
             render_template(
                 template_name,

doc/source/conf.py

@@ -134,7 +134,7 @@ html_static_path = ['_static']
 git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
            "-n1"]
 html_last_updated_fmt = subprocess.Popen(
-    git_cmd, stdout=subprocess.PIPE).communicate()[0]
+    git_cmd, stdout=subprocess.PIPE).communicate()[0].decode('utf-8')
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.

openstack_election/cmds/check_manual.py

@@ -41,9 +41,10 @@ def main():
         args.limit = 100
 
     projects = utils.get_projects(tag=args.tag)
-    if args.project_name not in projects.keys():
+    projects_list = list(projects.keys())
+    if args.project_name not in projects_list:
         print('[E]: %s not found in: %s' %
-              (args.project_name, ','.join(projects.keys())))
+              (args.project_name, ','.join(projects_list)))
         return 1
 
     if check_candidacy.check_candidate(args.project_name, args.email,

openstack_election/cmds/create_directories.py

@@ -30,7 +30,7 @@ def main():
         exit(1)
 
     projects = utils.get_projects()
-    project_list = projects.keys()
+    project_list = list(projects.keys())
     project_list.sort()
     for project in project_list + ["TC"]:
         dpath = "%s/%s" % (base_dir, utils.name2dir(project))

openstack_election/cmds/render_statistics.py

@@ -45,7 +45,7 @@ def collect_project_stats(basedir, verbose):
         project = directory[len(basedir):]
         if project == "TC":
             continue
-        candidates = filter(lambda x: x.endswith('.txt'), filenames)
+        candidates = list(filter(lambda x: x.endswith('.txt'), filenames))
         candidates_count = len(candidates)
 
         if not filenames == []:

openstack_election/utils.py

@@ -21,11 +21,14 @@ import pickle
 import pytz
 import re
 import requests
+import six
 import subprocess
 import time
-import urllib
 import yaml
 
+from six.moves.urllib.parse import quote_plus
+from six.moves.urllib.request import urlopen
+
 from openstack_election import config
 
 
@@ -110,7 +113,7 @@ def get_reviews(query):
     opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS']
     opts_str = '&o=%s' % ('&o='.join(opts))
     url = ('%s/changes/?q=%s%s' %
-           (GERRIT_BASE, urllib.quote_plus(query, safe='/:=><^.*'), opts_str))
+           (GERRIT_BASE, quote_plus(query, safe='/:=><^.*'), opts_str))
     return gerrit_query(url)
 
 
@@ -141,10 +144,10 @@ def get_projects(tag=None):
             os.stat(cache_file).st_size < 100 or
             os.stat(cache_file).st_mtime + (7*24*3600) < time.time()):
         print("[+] Updating %s" % (cache_file))
-        data = yaml.safe_load(urllib.urlopen(url).read())
+        data = yaml.safe_load(urlopen(url).read())
-        pickle.dump(data, open(cache_file, "w"))
+        pickle.dump(data, open(cache_file, "wb"), protocol=2)
 
-    return pickle.load(open(cache_file))
+    return pickle.load(open(cache_file, "rb"))
 
 
 # Election functions
@@ -176,7 +179,7 @@ def build_candidates_list(election=conf['release']):
         project_prefix = os.path.join(CANDIDATE_PATH, election, project)
         file_list = filter(
             lambda x: x.endswith(".txt"),
-            os.listdir(unicode(project_prefix)),
+            os.listdir(project_prefix),
         )
         candidates_list = []
         for candidate_file in file_list:
@@ -185,7 +188,7 @@ def build_candidates_list(election=conf['release']):
                 {
                     'url': ('%s/%s/plain/%s' %
                             (CGIT_URL, ELECTION_REPO,
-                             urllib.quote_plus(filepath, safe='/'))),
+                             quote_plus(filepath, safe='/'))),
                     'ircname': candidate_file[:-4].replace('`', r'\`'),
                     'email': get_email(filepath),
                     'fullname': get_fullname(filepath)

requirements.txt

@@ -1,6 +1,7 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
+six>=1.9.0 # MIT
 pytz>=2013.6 # MIT
 PyYAML>=3.1.0 # MIT
 requests>=2.10.0 # Apache-2.0

tox.ini

@@ -21,6 +21,14 @@ commands = yamllint configuration.yaml
 [testenv:venv]
 commands = {posargs}
 
+[testenv:venv2]
+basepython = python2
+commands = {posargs}
+
+[testenv:venv3]
+basepython = python3
+commands = {posargs}
+
 [testenv:docs]
 commands = python setup.py build_sphinx