trove/tools/trove-pylint.py

351 lines
11 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# Copyright 2016 Tesora, Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fnmatch
import json
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
from collections import OrderedDict
import io
import os
import re
import sys
from pylint import lint
from pylint.reporters import text
DEFAULT_CONFIG_FILE = "tools/trove-pylint.config"
DEFAULT_IGNORED_FILES = ['trove/tests']
DEFAULT_IGNORED_CODES = []
DEFAULT_IGNORED_MESSAGES = []
DEFAULT_ALWAYS_ERROR = [
"Undefined variable '_'",
"Undefined variable '_LE'",
"Undefined variable '_LI'",
"Undefined variable '_LW'",
"Undefined variable '_LC'"]
MODE_CHECK = "check"
MODE_REBUILD = "rebuild"
class Config(object):
def __init__(self, filename=DEFAULT_CONFIG_FILE):
self.default_config = {
"include": ["*.py"],
"folder": "trove",
"options": ["--rcfile=./pylintrc", "-E"],
"ignored_files": DEFAULT_IGNORED_FILES,
"ignored_codes": DEFAULT_IGNORED_CODES,
"ignored_messages": DEFAULT_IGNORED_MESSAGES,
"ignored_file_codes": [],
"ignored_file_messages": [],
"ignored_file_code_messages": [],
"always_error_messages": DEFAULT_ALWAYS_ERROR
}
self.config = self.default_config
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
def sort_config(self):
sorted_config = OrderedDict()
for key in sorted(self.config.keys()):
value = self.get(key)
if isinstance(value, list) and not isinstance(value,str):
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
sorted_config[key] = sorted(value)
else:
sorted_config[key] = value
return sorted_config
def save(self, filename=DEFAULT_CONFIG_FILE):
if os.path.isfile(filename):
os.rename(filename, "%s~" % filename)
with open(filename, 'w') as fp:
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
json.dump(self.sort_config(), fp, encoding="utf-8",
indent=2, separators=(',', ': '))
def load(self, filename=DEFAULT_CONFIG_FILE):
with open(filename) as fp:
self.config = json.load(fp, encoding="utf-8")
def get(self, attribute):
return self.config[attribute]
def is_file_ignored(self, f):
if any(f.startswith(i)
for i in self.config['ignored_files']):
return True
return False
def is_file_included(self, f):
if any(fnmatch.fnmatch(f, wc) for wc in self.config['include']):
return True
return False
def is_always_error(self, message):
if message in self.config['always_error_messages']:
return True
return False
def ignore(self, filename, code, codename, message):
# the high priority checks
if self.is_file_ignored(filename):
return True
# never ignore messages
if self.is_always_error(message):
return False
if code in self.config['ignored_codes']:
return True
if codename in self.config['ignored_codes']:
return True
if message and any(message.startswith(ignore_message)
for ignore_message
in self.config['ignored_messages']):
return True
if filename and message and (
[filename, message] in self.config['ignored_file_messages']):
return True
if filename and code and (
[filename, code] in self.config['ignored_file_codes']):
return True
if filename and codename and (
[filename, codename] in self.config['ignored_file_codes']):
return True
for fcm in self.config['ignored_file_code_messages']:
if filename != fcm[0]:
# This ignore rule is for a different file.
continue
if fcm[1] not in (code, codename):
# This ignore rule is for a different code or codename.
continue
if message.startswith(fcm[2]):
return True
return False
def ignore_code(self, c):
_c = set(self.config['ignored_codes'])
_c.add(c)
self.config['ignored_codes'] = list(_c)
def ignore_files(self, f):
_c = set(self.config['ignored_files'])
_c.add(f)
self.config['ignored_files'] = list(_c)
def ignore_message(self, m):
_c = set(self.config['ignored_messages'])
_c.add(m)
self.config['ignored_messages'] = list(_c)
def ignore_file_code(self, f, c):
_c = set(self.config['ignored_file_codes'])
_c.add((f, c))
self.config['ignored_file_codes'] = list(_c)
def ignore_file_message(self, f, m):
_c = set(self.config['ignored_file_messages'])
_c.add((f, m))
self.config['ignored_file_messages'] = list(_c)
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
def ignore_file_code_message(self, f, c, m, fn):
_c = set(self.config['ignored_file_code_messages'])
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
_c.add((f, c, m, fn))
self.config['ignored_file_code_messages'] = list(_c)
def main():
if len(sys.argv) == 1 or sys.argv[1] == "check":
return check()
elif sys.argv[1] == "rebuild":
return rebuild()
elif sys.argv[1] == "initialize":
return initialize()
else:
return usage()
def usage():
print("Usage: %s [check|rebuild]" % sys.argv[0])
print("\tUse this tool to perform a lint check of the trove project.")
print("\t check: perform the lint check.")
print("\t rebuild: rebuild the list of exceptions to ignore.")
return 0
class ParseableTextReporter(text.TextReporter):
name = 'parseable'
line_format = '{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'
# that's it folks
class LintRunner(object):
def __init__(self):
self.config = Config()
self.idline = re.compile("^[*]* Module .*")
self.detail = re.compile(r"(\S+):(\d+): \[(\S+)\((\S+)\),"
r" (\S+)?] (.*)")
def dolint(self, filename):
exceptions = set()
buffer = io.StringIO()
reporter = ParseableTextReporter(output=buffer)
options = list(self.config.get('options'))
options.append(filename)
lint.Run(options, reporter=reporter, exit=False)
output = buffer.getvalue()
buffer.close()
for line in output.splitlines():
if self.idline.match(line):
continue
if self.detail.match(line):
mo = self.detail.search(line)
tokens = mo.groups()
fn = tokens[0]
ln = tokens[1]
code = tokens[2]
codename = tokens[3]
func = tokens[4]
message = tokens[5]
if not self.config.ignore(fn, code, codename, message):
exceptions.add((fn, ln, code, codename, func, message))
return exceptions
def process(self, mode=MODE_CHECK):
files_processed = 0
files_with_errors = 0
errors_recorded = 0
exceptions_recorded = 0
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
all_exceptions = []
for (root, dirs, files) in os.walk(self.config.get('folder')):
# if we shouldn't even bother about this part of the
# directory structure, we can punt quietly
if self.config.is_file_ignored(root):
continue
# since we are walking top down, let's clean up the dirs
# that we will walk by eliminating any dirs that will
# end up getting ignored
for d in dirs:
p = os.path.join(root, d)
if self.config.is_file_ignored(p):
dirs.remove(d)
# check if we can ignore the file and process if not
for f in files:
p = os.path.join(root, f)
if self.config.is_file_ignored(p):
continue
if not self.config.is_file_included(f):
continue
files_processed += 1
exceptions = self.dolint(p)
file_had_errors = 0
for e in exceptions:
# what we do with this exception depents on the
# kind of exception, and the mode
if self.config.is_always_error(e[5]):
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
all_exceptions.append(e)
errors_recorded += 1
file_had_errors += 1
elif mode == MODE_REBUILD:
# parameters to ignore_file_code_message are
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
# filename, code, message and function
self.config.ignore_file_code_message(e[0], e[2], e[-1], e[4])
self.config.ignore_file_code_message(e[0], e[3], e[-1], e[4])
exceptions_recorded += 1
elif mode == MODE_CHECK:
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
all_exceptions.append(e)
errors_recorded += 1
file_had_errors += 1
if file_had_errors:
files_with_errors += 1
improve pylint; generate errors and config in sorted order Since the config is not in a deterministic order makes it hard to compare two config's and see what changed. Personally, I'm not positive I understand this use-case; i.e. you have an existing config file, you save it, and then rebuild and then diff the two files. I'd have thought you'd just run check and the output of the tool was the diff. I however do see the value in sorting the file so that when someone submits a change that includes a change to the config, reviewers can see more easily what the change is doing. Similarly, the output from pylint (errors) are generated one file at a time and os.walk makes no guarantee of deterministic order. So we should collect all errors (across all files) and then print an ordered list for human consumption. The intent is also to make pylint voting soon (in master). the changes to contributing.rst and tox.ini are to make that easier. The config file has also been sorted in place. This change was motivated by an email exchange with Peter so I am marking him as a co-conspirator. The line numbers were removed from the tools/trove-pylint.config file as these would change whenever the line numbers in the file changed (since they are currently not being used in the comparison; they can be re-added if deemed necessary at the cost of having every 'rebuild' run create a different file). The tools/trove-pylint.config was regenerated as well, since the remaining two errors seem to be innocuous: ERROR: trove/taskmanager/manager.py 392: E1101 no-member, Manager.upgrade: Instance of 'BuiltInstance' has no 'upgrade' member (new method introduced by instance upgrade; other BuiltInstance member errors are already ignored.) and ERROR: trove/guestagent/datastore/experimental/postgresql/service/ access.py 80: E1101 no-member, PgSqlAccess.list_access: Instance ofi 'PgSqlAccess' has no '_find_user' member (this is due to the fact that PostgreSQL is spread over multiple files and pylint should cease to complain once https://review.openstack.org/#/c/346082/ lands.) Change-Id: I910c738d3845b7749e57910f76523150ec5a5bff Closes-Bug: #1625158 Closes-Bug: #1625245 Co-Authored-By: Peter Stachowski <peter@tesora.com>
2016-09-19 08:26:24 -04:00
for e in sorted(all_exceptions):
print("ERROR: %s %s: %s %s, %s: %s" %
(e[0], e[1], e[2], e[3], e[4], e[5]))
return (files_processed, files_with_errors, errors_recorded,
exceptions_recorded)
def rebuild(self):
self.initialize()
(files_processed,
files_with_errors,
errors_recorded,
exceptions_recorded) = self.process(mode=MODE_REBUILD)
if files_with_errors > 0:
print("Rebuild failed. %s files processed, %s had errors, "
"%s errors recorded." % (
files_processed, files_with_errors, errors_recorded))
return 1
self.config.save()
print("Rebuild completed. %s files processed, %s exceptions recorded." %
(files_processed, exceptions_recorded))
return 0
def check(self):
self.config.load()
(files_processed,
files_with_errors,
errors_recorded,
exceptions_recorded) = self.process(mode=MODE_CHECK)
if files_with_errors > 0:
print("Check failed. %s files processed, %s had errors, "
"%s errors recorded." % (
files_processed, files_with_errors, errors_recorded))
return 1
print("Check succeeded. %s files processed" % files_processed)
return 0
def initialize(self):
self.config.save()
return 0
def check():
exit(LintRunner().check())
def rebuild():
exit(LintRunner().rebuild())
def initialize():
exit(LintRunner().initialize())
if __name__ == "__main__":
main()