From 233a61d1e0343c042502afe59f3e27ccdc4ca130 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 11 Sep 2020 07:57:31 -0500 Subject: [PATCH] [CI] Fix gate - multiple fixes (Focal/Py38 and policy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppress of policy deprecation warnings. oslo.policy logs the deprecated policy warnings while loading the policy rules. Flag to suppress the warning was set after load_rule() was called. We do not need to log warnings for unit test where every test initializes the policy and logs a warning. Also use yaml format of policy file in unit test to avoid more warnings for json file. Bump a few l-c for Py38 testing, squashing parts of https://review.opendev.org/744343 Fix unit tests compatibility with Py38, squashing parts of https://review.opendev.org/748874 Change-Id: Ia54d29975347392cbb93c07969d7e3b48eca5d23 Co-Authored-By: Radosław Piliszek --- lower-constraints.txt | 10 +++++----- masakari/policy.py | 14 +++++++++++++- masakari/tests/unit/policy_fixture.py | 16 ++++++++-------- masakari/tests/unit/test_api_validation.py | 6 +----- masakari/tests/unit/test_policy.py | 6 +++--- requirements.txt | 2 +- test-requirements.txt | 2 +- 7 files changed, 32 insertions(+), 24 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 3b703e2e..12ebcb69 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -31,10 +31,10 @@ Jinja2==2.10 jsonschema==3.2.0 keystoneauth1==3.4.0 keystonemiddleware==4.17.0 -kombu==4.1.0 +kombu==4.6.1 linecache2==1.0.0 Mako==1.0.7 -MarkupSafe==1.0 +MarkupSafe==1.1.1 microversion-parse==0.2.1 mox3==0.25.0 msgpack==0.5.6 @@ -48,7 +48,7 @@ oslo.cache==1.29.0 oslo.concurrency==3.26.0 oslo.config==5.2.0 oslo.context==2.19.2 -oslo.db==4.27.0 +oslo.db==4.44.0 oslo.i18n==3.15.3 oslo.log==3.36.0 oslo.messaging==5.29.0 @@ -67,7 +67,7 @@ pbr==2.0.0 pika-pool==0.1.3 pika==0.10.0 prettytable==0.7.2 -psycopg2==2.7 +psycopg2==2.8 pycadf==2.7.0 Pygments==2.2.0 pyinotify==0.9.6 @@ -81,7 +81,7 @@ python-mimeparse==1.6.0 python-novaclient==9.1.0 python-subunit==1.0.0 pytz==2018.3 -PyYAML==3.12 +PyYAML==3.13 repoze.lru==0.7 requests-mock==1.2.0 requests==2.18.4 diff --git a/masakari/policy.py b/masakari/policy.py index 0b9d485d..569d7b3c 100644 --- a/masakari/policy.py +++ b/masakari/policy.py @@ -45,7 +45,8 @@ def reset(): _ENFORCER = None -def init(policy_file=None, rules=None, default_rule=None, use_conf=True): +def init(policy_file=None, rules=None, default_rule=None, use_conf=True, + suppress_deprecation_warnings=False): """Init an Enforcer class. :param policy_file: Custom policy file to use, if none is specified, @@ -55,6 +56,8 @@ def init(policy_file=None, rules=None, default_rule=None, use_conf=True): :param default_rule: Default rule to use, CONF.default_rule will be used if none is specified. :param use_conf: Whether to load rules from config file. + :param suppress_deprecation_warnings: Whether to suppress the + deprecation warnings. """ global _ENFORCER global saved_file_rules @@ -64,6 +67,15 @@ def init(policy_file=None, rules=None, default_rule=None, use_conf=True): rules=rules, default_rule=default_rule, use_conf=use_conf) + # NOTE(gmann): Explictly disable the warnings for policies + # changing their default check_str. During policy-defaults-refresh + # work, all the policy defaults have been changed and warning for + # each policy started filling the logs limit for various tool. + # Once we move to new defaults only world then we can enable these + # warning again. + _ENFORCER.suppress_default_change_warnings = True + if suppress_deprecation_warnings: + _ENFORCER.suppress_deprecation_warnings = True register_rules(_ENFORCER) _ENFORCER.load_rules() diff --git a/masakari/tests/unit/policy_fixture.py b/masakari/tests/unit/policy_fixture.py index 5f8aa21c..4fdfc31c 100644 --- a/masakari/tests/unit/policy_fixture.py +++ b/masakari/tests/unit/policy_fixture.py @@ -13,10 +13,10 @@ # under the License. import os +import yaml import fixtures from oslo_policy import policy as oslo_policy -from oslo_serialization import jsonutils import masakari.conf from masakari.conf import paths @@ -46,11 +46,11 @@ class RealPolicyFixture(fixtures.Fixture): def setUp(self): super(RealPolicyFixture, self).setUp() # policy_file can be overridden by subclasses - self.policy_file = paths.state_path_def('etc/masakari/policy.json') + self.policy_file = paths.state_path_def('etc/masakari/policy.yaml') self._prepare_policy() CONF.set_override('policy_file', self.policy_file, group='oslo_policy') masakari.policy.reset() - masakari.policy.init() + masakari.policy.init(suppress_deprecation_warnings=True) self.addCleanup(masakari.policy.reset) def set_rules(self, rules, overwrite=True): @@ -68,13 +68,13 @@ class PolicyFixture(RealPolicyFixture): """ def _prepare_policy(self): self.policy_dir = self.useFixture(fixtures.TempDir()) - self.policy_file = os.path.join(self.policy_dir.path, 'policy.json') + self.policy_file = os.path.join(self.policy_dir.path, 'policy.yaml') # load the fake_policy data and add the missing default rules. - policy_rules = jsonutils.loads(fake_policy.policy_data) + policy_rules = yaml.safe_load(fake_policy.policy_data) with open(self.policy_file, 'w') as f: - jsonutils.dump(policy_rules, f) + yaml.dump(policy_rules, f) CONF.set_override('policy_dirs', [], group='oslo_policy') @@ -99,6 +99,6 @@ class RoleBasedPolicyFixture(RealPolicyFixture): policy[rule.name] = 'role:%s' % self.role self.policy_dir = self.useFixture(fixtures.TempDir()) - self.policy_file = os.path.join(self.policy_dir.path, 'policy.json') + self.policy_file = os.path.join(self.policy_dir.path, 'policy.yaml') with open(self.policy_file, 'w') as f: - jsonutils.dump(policy, f) + yaml.dump(policy, f) diff --git a/masakari/tests/unit/test_api_validation.py b/masakari/tests/unit/test_api_validation.py index 2865f9b6..b0d9084c 100644 --- a/masakari/tests/unit/test_api_validation.py +++ b/masakari/tests/unit/test_api_validation.py @@ -18,7 +18,6 @@ import re import fixtures from jsonschema import exceptions as jsonschema_exc -import sys from masakari.api import api_version_request as api_version from masakari.api import validation @@ -269,10 +268,7 @@ class PatternPropertiesTestCase(APIValidationTestCase): self.check_validation_error(self.post, body={'0123456789a': 'bar'}, expected_detail=details) - if sys.version[:3] in ['3.5', '3.6', '3.7']: - detail = "expected string or bytes-like object" - else: - detail = "expected string or buffer" + detail = "expected string or bytes-like object" self.check_validation_error(self.post, body={None: 'bar'}, expected_detail=detail) diff --git a/masakari/tests/unit/test_policy.py b/masakari/tests/unit/test_policy.py index 71b41194..f363aae2 100644 --- a/masakari/tests/unit/test_policy.py +++ b/masakari/tests/unit/test_policy.py @@ -49,7 +49,7 @@ class PolicyFileTestCase(test.NoDBTestCase): # determine is_admin or not. As a side-effect, policy reset is # needed here to flush existing policy cache. policy.reset() - policy.init() + policy.init(suppress_deprecation_warnings=True) rule = oslo_policy.RuleDefault('example:test', "") policy._ENFORCER.register_defaults([rule]) @@ -84,7 +84,7 @@ class PolicyTestCase(test.NoDBTestCase): "role:ADMIN or role:sysadmin"), ] policy.reset() - policy.init() + policy.init(suppress_deprecation_warnings=True) # before a policy rule can be used, its default has to be registered. policy._ENFORCER.register_defaults(rules) self.context = context.RequestContext('fake', 'fake', roles=['member']) @@ -158,7 +158,7 @@ class PolicyTestCase(test.NoDBTestCase): class IsAdminCheckTestCase(test.NoDBTestCase): def setUp(self): super(IsAdminCheckTestCase, self).setUp() - policy.init() + policy.init(suppress_deprecation_warnings=True) def test_init_true(self): check = policy.IsAdminCheck('is_admin', 'True') diff --git a/requirements.txt b/requirements.txt index 1a9e1b64..5e9b5862 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ WebOb>=1.7.1 # MIT microversion-parse>=0.2.1 # Apache-2.0 oslo.config>=5.2.0 # Apache-2.0 oslo.context>=2.19.2 # Apache-2.0 -oslo.db>=4.27.0 # Apache-2.0 +oslo.db>=4.44.0 # Apache-2.0 oslo.messaging>=5.29.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 oslo.log>=3.36.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 02bdc534..fd1e388f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,7 +7,7 @@ hacking>=3.0.1,<3.1.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 ddt>=1.0.1 # MIT pep8>=1.5.7 -psycopg2>=2.7 # LGPL/ZPL +psycopg2>=2.8 # LGPL/ZPL PyMySQL>=0.7.6 # MIT License python-subunit>=1.0.0 # Apache-2.0/BSD openstacksdk>=0.35.0 # Apache-2.0