From a3ac4a90f89d07fc7ea4073184f7f0c6321febf4 Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Wed, 4 Jan 2017 16:28:24 +1100 Subject: [PATCH] Support for running tools under python3 If we run the tools (create-directories or check-all-candidacies) under python3 they'll fail with silly errors (like[1]). This is because we have a number of assumptions they we'll be running under python2 like: - pickle.protocol: we done specify one so if we write a pickle file from python3 we can't read it in python2. - urllib and urlparse: those modules change paths in python 2 or 3 so use the six helpers to get the bits we want. - Explicitly write files in binary format - Various str vs bytes issues - dict().keys() returns dict_keys under python3 explictly cast that to list() After addressing these issues we can run the tools under either python2 or python3. To make this easy to check create explicit ven2 and venv3 test environments, leaving all other testenvs to use the system default. [1] http://paste.openstack.org/show/593823/ Change-Id: I20334a52500847c810b486c9c8b108e75a5d6303 --- doc/source/_exts/candidates.py | 4 ++-- doc/source/conf.py | 2 +- openstack_election/cmds/check_manual.py | 5 +++-- openstack_election/cmds/create_directories.py | 2 +- openstack_election/cmds/render_statistics.py | 2 +- openstack_election/utils.py | 17 ++++++++++------- requirements.txt | 1 + tox.ini | 8 ++++++++ 8 files changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/source/_exts/candidates.py b/doc/source/_exts/candidates.py index 8cfb5553..2ab9ea03 100644 --- a/doc/source/_exts/candidates.py +++ b/doc/source/_exts/candidates.py @@ -33,7 +33,7 @@ def render_list(list_type, candidates_list): output_file = os.path.join(".", "doc", "source", "%s.rst" % list_type) template_name = "%s.jinja" % list_type template_dir = os.path.join(".", "doc", "source", "_exts") - with open(output_file, "w") as out: + with open(output_file, "wb") as out: out.write( render_template( template_name, @@ -59,7 +59,7 @@ def build_archive(serie, list_type): output = os.path.join(".", "doc", "source", serie, "%s.rst" % list_type) template_name = "%s_archive.jinja" % list_type template_dir = os.path.join(".", "doc", "source", "_exts") - with open(output, "w") as out: + with open(output, "wb") as out: out.write( render_template( template_name, diff --git a/doc/source/conf.py b/doc/source/conf.py index 3eb39344..0551bba8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -134,7 +134,7 @@ html_static_path = ['_static'] git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local", "-n1"] html_last_updated_fmt = subprocess.Popen( - git_cmd, stdout=subprocess.PIPE).communicate()[0] + git_cmd, stdout=subprocess.PIPE).communicate()[0].decode('utf-8') # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/openstack_election/cmds/check_manual.py b/openstack_election/cmds/check_manual.py index 95a6aedd..fd42765b 100644 --- a/openstack_election/cmds/check_manual.py +++ b/openstack_election/cmds/check_manual.py @@ -41,9 +41,10 @@ def main(): args.limit = 100 projects = utils.get_projects(tag=args.tag) - if args.project_name not in projects.keys(): + projects_list = list(projects.keys()) + if args.project_name not in projects_list: print('[E]: %s not found in: %s' % - (args.project_name, ','.join(projects.keys()))) + (args.project_name, ','.join(projects_list))) return 1 if check_candidacy.check_candidate(args.project_name, args.email, diff --git a/openstack_election/cmds/create_directories.py b/openstack_election/cmds/create_directories.py index ed6654de..d888c15b 100755 --- a/openstack_election/cmds/create_directories.py +++ b/openstack_election/cmds/create_directories.py @@ -30,7 +30,7 @@ def main(): exit(1) projects = utils.get_projects() - project_list = projects.keys() + project_list = list(projects.keys()) project_list.sort() for project in project_list + ["TC"]: dpath = "%s/%s" % (base_dir, utils.name2dir(project)) diff --git a/openstack_election/cmds/render_statistics.py b/openstack_election/cmds/render_statistics.py index 44f3699d..98627502 100755 --- a/openstack_election/cmds/render_statistics.py +++ b/openstack_election/cmds/render_statistics.py @@ -45,7 +45,7 @@ def collect_project_stats(basedir, verbose): project = directory[len(basedir):] if project == "TC": continue - candidates = filter(lambda x: x.endswith('.txt'), filenames) + candidates = list(filter(lambda x: x.endswith('.txt'), filenames)) candidates_count = len(candidates) if not filenames == []: diff --git a/openstack_election/utils.py b/openstack_election/utils.py index 0e2c78d4..084b2740 100644 --- a/openstack_election/utils.py +++ b/openstack_election/utils.py @@ -21,11 +21,14 @@ import pickle import pytz import re import requests +import six import subprocess import time -import urllib import yaml +from six.moves.urllib.parse import quote_plus +from six.moves.urllib.request import urlopen + from openstack_election import config @@ -110,7 +113,7 @@ def get_reviews(query): opts = ['CURRENT_REVISION', 'CURRENT_FILES', 'DETAILED_ACCOUNTS'] opts_str = '&o=%s' % ('&o='.join(opts)) url = ('%s/changes/?q=%s%s' % - (GERRIT_BASE, urllib.quote_plus(query, safe='/:=><^.*'), opts_str)) + (GERRIT_BASE, quote_plus(query, safe='/:=><^.*'), opts_str)) return gerrit_query(url) @@ -141,10 +144,10 @@ def get_projects(tag=None): os.stat(cache_file).st_size < 100 or os.stat(cache_file).st_mtime + (7*24*3600) < time.time()): print("[+] Updating %s" % (cache_file)) - data = yaml.safe_load(urllib.urlopen(url).read()) - pickle.dump(data, open(cache_file, "w")) + data = yaml.safe_load(urlopen(url).read()) + pickle.dump(data, open(cache_file, "wb"), protocol=2) - return pickle.load(open(cache_file)) + return pickle.load(open(cache_file, "rb")) # Election functions @@ -176,7 +179,7 @@ def build_candidates_list(election=conf['release']): project_prefix = os.path.join(CANDIDATE_PATH, election, project) file_list = filter( lambda x: x.endswith(".txt"), - os.listdir(unicode(project_prefix)), + os.listdir(project_prefix), ) candidates_list = [] for candidate_file in file_list: @@ -185,7 +188,7 @@ def build_candidates_list(election=conf['release']): { 'url': ('%s/%s/plain/%s' % (CGIT_URL, ELECTION_REPO, - urllib.quote_plus(filepath, safe='/'))), + quote_plus(filepath, safe='/'))), 'ircname': candidate_file[:-4].replace('`', r'\`'), 'email': get_email(filepath), 'fullname': get_fullname(filepath) diff --git a/requirements.txt b/requirements.txt index 7dd88b4a..7ae98c6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +six>=1.9.0 # MIT pytz>=2013.6 # MIT PyYAML>=3.1.0 # MIT requests>=2.10.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 4fa3aac7..c0f36b50 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,14 @@ commands = yamllint configuration.yaml [testenv:venv] commands = {posargs} +[testenv:venv2] +basepython = python2 +commands = {posargs} + +[testenv:venv3] +basepython = python3 +commands = {posargs} + [testenv:docs] commands = python setup.py build_sphinx