Merge "Rewrite generate-rolls in python"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
74a076057c
113
openstack_election/cmds/generate_rolls.py
Executable file
113
openstack_election/cmds/generate_rolls.py
Executable file
@ -0,0 +1,113 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from six.moves.urllib.request import urlopen
|
||||
|
||||
from openstack_election import utils
|
||||
|
||||
CGIT_URL = "http://git.openstack.org/cgit/openstack-infra/system-config/" \
|
||||
"plain/tools/owners.py"
|
||||
|
||||
# Exclude the system / bot accounts
|
||||
# OpenStack Release Bot:
|
||||
# curl https://review.openstack.org/accounts/22816
|
||||
# OpenStack Proposal Bot
|
||||
# curl https://review.openstack.org/accounts/11131
|
||||
MINUS_BOT_ACCOUNTS = ["-i", "11131", "-i", "22816"]
|
||||
|
||||
|
||||
def main():
|
||||
start = utils.conf['timeframe']['start']
|
||||
end = utils.conf['timeframe']['end']
|
||||
description = ('Generate electorate rolls')
|
||||
parser = argparse.ArgumentParser(description)
|
||||
parser.add_argument('--tag', dest='tag', default=utils.conf['tag'],
|
||||
help=('The governance tag to validate against. '
|
||||
'Default: %(default)s'))
|
||||
parser.add_argument('--rolls_dir', default="./rolls/%s" % (
|
||||
utils.conf['tag']), help=('The output directory. '
|
||||
'Default: %(default)s'))
|
||||
parser.add_argument('--before', default=end.strftime("%Y-%m-%d %H-%M-%S"),
|
||||
help=('The start date. Default: %(default)s'))
|
||||
parser.add_argument('--after', default=start.strftime("%Y-%m-%d"),
|
||||
help=('The end date. Default: %(default)s'))
|
||||
parser.add_argument('--without-stable', dest='with_stable', default=True,
|
||||
action='store_false', help=('Do not include stables'))
|
||||
parser.add_argument('--cached-owners-script', default=False,
|
||||
action='store_true', help=('Do not fetch the script'))
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Run information:")
|
||||
print(" - TAG : %s" % args.tag)
|
||||
print(" - ROLLS : %s" % args.rolls_dir)
|
||||
print(" - STABLE : %s" % args.with_stable)
|
||||
print(" - BEFORE : %s" % args.before)
|
||||
print(" - AFTER : %s" % args.after)
|
||||
|
||||
if os.path.isdir(args.rolls_dir):
|
||||
shutil.rmtree(args.rolls_dir)
|
||||
os.makedirs(args.rolls_dir, 0o700)
|
||||
|
||||
os.chdir(os.path.dirname(args.rolls_dir))
|
||||
if not args.cached_owners_script:
|
||||
print("Grabbing script from: %s" % CGIT_URL)
|
||||
with open("owners.py", "wb") as owner_file:
|
||||
owner_file.write(urlopen(CGIT_URL).read())
|
||||
|
||||
sys.path.append(os.getcwd())
|
||||
import owners
|
||||
print("Starting roll generation @%s" % time.ctime())
|
||||
owners.main(["owners.py", "-a", args.after, "-b", args.before,
|
||||
"-o", args.tag, "-r", args.tag] + MINUS_BOT_ACCOUNTS)
|
||||
print("Finished roll generation @%s" % time.ctime())
|
||||
|
||||
if args.with_stable:
|
||||
tmp_dir = tempfile.mkdtemp(prefix='election.')
|
||||
print("Starting (Stable) roll generation @%s" % time.ctime())
|
||||
owners.main(["owners.py", "-a", args.after, "-b", args.before,
|
||||
"-o", tmp_dir, "-r", args.tag, "-n",
|
||||
"-s", "branch:^stable/.*"] + MINUS_BOT_ACCOUNTS)
|
||||
print("Finished (Stable) roll generation @%s" % time.ctime())
|
||||
shutil.copy("%s/_electorate.txt" % tmp_dir,
|
||||
"./%s/stable_branch_maintenance.txt" % args.tag)
|
||||
shutil.copy("%s/_all_owners.yaml" % tmp_dir,
|
||||
"./%s/stable_branch_maintenance.yaml" % args.tag)
|
||||
shutil.rmtree(tmp_dir)
|
||||
|
||||
print("Electoral rolls SHA256sum")
|
||||
with open("%s.shasums" % args.tag, "wb") as ofile:
|
||||
for fname in sorted(glob.glob("./%s/*.txt" % args.tag)):
|
||||
# Sort the list
|
||||
with open(fname) as list_file:
|
||||
roll = sorted(list_file.readlines())
|
||||
content = "".join(roll).encode('utf-8')
|
||||
with open(fname, "wb") as list_file:
|
||||
list_file.write(content)
|
||||
# Generate the hash
|
||||
h = hashlib.sha256(content).hexdigest()
|
||||
s = "%s %s" % (h, fname)
|
||||
print(s)
|
||||
ofile.write(("%s\n" % s).encode('utf-8'))
|
||||
@ -28,6 +28,7 @@ console_scripts =
|
||||
close-election = openstack_election.cmds.close_election:main
|
||||
search-rolls = openstack_election.cmds.search_rolls:main
|
||||
create-directories = openstack_election.cmds.create_directories:main
|
||||
generate-rolls = openstack_election.cmds.generate_rolls:main
|
||||
|
||||
[build_sphinx]
|
||||
all_files = 1
|
||||
|
||||
@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CGIT_URL=http://git.openstack.org/cgit/openstack-infra/system-config/plain/tools/owners.py
|
||||
OWNERS=$(basename "$CGIT_URL")
|
||||
WITH_STABLE=1
|
||||
|
||||
while [ $# -gt 0 ] ; do
|
||||
case "$1" in
|
||||
--tag)
|
||||
TAG=$2
|
||||
shift 1
|
||||
;;
|
||||
--before)
|
||||
BEFORE=$2
|
||||
shift 1
|
||||
;;
|
||||
--after)
|
||||
AFTER=$2
|
||||
shift 1
|
||||
;;
|
||||
--without-stable)
|
||||
WITH_STABLE=0
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
shift 1
|
||||
done
|
||||
|
||||
if [ -z "$TAG" ] ; then
|
||||
echo $(basename "$0"): You must supply a --tag >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$AFTER" ] ; then
|
||||
echo $(basename "$0"): You must supply a --after >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$BEFORE" ] ; then
|
||||
echo $(basename "$0"): You must supply a --before >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
ROLLS_DIR="./rolls/${TAG}"
|
||||
|
||||
echo "Run information:"
|
||||
echo " - TAG :" $TAG
|
||||
echo " - ROLLS :" $ROLLS_DIR
|
||||
echo " - STABLE :" $WITH_STABLE
|
||||
echo " - AFTER :" $AFTER
|
||||
echo " - BEFORE :" $BEFORE
|
||||
|
||||
rm -rf "${ROLLS_DIR}"
|
||||
mkdir -p "${ROLLS_DIR}"
|
||||
|
||||
cd $(dirname "${ROLLS_DIR}")
|
||||
|
||||
# Building the venv isn't really needed everytime
|
||||
if [ ! -d venv ] ; then
|
||||
echo "Building tools venv"
|
||||
(
|
||||
# owners.py isn't python3 safe as string.maketrans is now
|
||||
# str.maketrans
|
||||
virtualenv -p python2 venv
|
||||
./venv/bin/pip install -U pip setuptools wheel
|
||||
./venv/bin/pip install pyyaml requests ndg-httpsclient
|
||||
) >/dev/null 2>&1
|
||||
if [ $? -ne 0 ] ; then
|
||||
echo Creating Virtualenv failed >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Using existing venv"
|
||||
fi
|
||||
|
||||
# Always get the newest script in case a bug has been fixed.
|
||||
rm -f "./${OWNERS}" >/dev/null 2>&1
|
||||
echo "Grabbing script from: ${CGIT_URL}"
|
||||
wget "${CGIT_URL}" -O "./${OWNERS}" -o /dev/null
|
||||
if [ ! -e "./${OWNERS}" ] ; then
|
||||
echo Failed to download ${CGIT_URL} >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting roll generation @" $(TZ=UTC date)
|
||||
# Exclude the system / bot accounts
|
||||
# OpenStack Release Bot:
|
||||
## curl https://review.openstack.org/accounts/22816
|
||||
# OpenStack Proposal Bot
|
||||
## curl https://review.openstack.org/accounts/11131
|
||||
venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \
|
||||
-i 11131 -i 22816 \
|
||||
-o "${TAG}" -r "${TAG}"
|
||||
if [ "$WITH_STABLE" -eq 1 ] ; then
|
||||
TMP_DIR=$(mktemp -d election.XXXXXXX)
|
||||
echo "Starting (Stable) roll generation @" $(TZ=UTC date)
|
||||
venv/bin/python "./${OWNERS}" -a "${AFTER}" -b "${BEFORE}" \
|
||||
-i 11131 -i 22816 \
|
||||
-o "${TMP_DIR}" -r "${TAG}" \
|
||||
-n -s 'branch:^stable/.*'
|
||||
find
|
||||
cp "${TMP_DIR}/_electorate.txt" "./${TAG}/stable_branch_maintenance.txt"
|
||||
cp "${TMP_DIR}/_all_owners.yaml" "./${TAG}/stable_branch_maintenance.yaml"
|
||||
echo "Finished (Stable) roll generation @" $(TZ=UTC date)
|
||||
rm -rf "${TMP_DIR}"
|
||||
else
|
||||
echo Dropping stable_branch_maintenance data
|
||||
rm "./${TAG}/stable_branch_maintenance.txt" \
|
||||
"./${TAG}/stable_branch_maintenance.yaml"
|
||||
fi
|
||||
echo "Finished roll generation @" $(TZ=UTC date)
|
||||
|
||||
echo "Electoral rolls SHA256sum"
|
||||
case "$(uname -s)" in
|
||||
Darwin)
|
||||
shasum -a 256 ./${TAG}/*.txt | tee "${TAG}.shasums"
|
||||
;;
|
||||
*)
|
||||
sha256sum ./${TAG}/*.txt | tee "${TAG}.shasums"
|
||||
;;
|
||||
esac
|
||||
Loading…
Reference in New Issue
Block a user