[CI] Fix gate - multiple fixes (Focal/Py38 and policy)

Suppress of policy deprecation warnings. oslo.policy logs the
deprecated policy warnings while loading the
policy rules. Flag to suppress the warning was set after load_rule()
was called. We do not need to log warnings for unit test where
every test initializes the policy and logs a warning.

Also use yaml format of policy file in unit test to avoid
more warnings for json file.

Bump a few l-c for Py38 testing, squashing parts of
https://review.opendev.org/744343

Fix unit tests compatibility with Py38, squashing parts of
https://review.opendev.org/748874

Change-Id: Ia54d29975347392cbb93c07969d7e3b48eca5d23
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>

lower-constraints.txt

@@ -31,10 +31,10 @@ Jinja2==2.10
 jsonschema==3.2.0
 keystoneauth1==3.4.0
 keystonemiddleware==4.17.0
-kombu==4.1.0
+kombu==4.6.1
 linecache2==1.0.0
 Mako==1.0.7
-MarkupSafe==1.0
+MarkupSafe==1.1.1
 microversion-parse==0.2.1
 mox3==0.25.0
 msgpack==0.5.6
@@ -48,7 +48,7 @@ oslo.cache==1.29.0
 oslo.concurrency==3.26.0
 oslo.config==5.2.0
 oslo.context==2.19.2
-oslo.db==4.27.0
+oslo.db==4.44.0
 oslo.i18n==3.15.3
 oslo.log==3.36.0
 oslo.messaging==5.29.0
@@ -67,7 +67,7 @@ pbr==2.0.0
 pika-pool==0.1.3
 pika==0.10.0
 prettytable==0.7.2
-psycopg2==2.7
+psycopg2==2.8
 pycadf==2.7.0
 Pygments==2.2.0
 pyinotify==0.9.6
@@ -81,7 +81,7 @@ python-mimeparse==1.6.0
 python-novaclient==9.1.0
 python-subunit==1.0.0
 pytz==2018.3
-PyYAML==3.12
+PyYAML==3.13
 repoze.lru==0.7
 requests-mock==1.2.0
 requests==2.18.4

masakari/policy.py

@@ -45,7 +45,8 @@ def reset():
         _ENFORCER = None
 
 
-def init(policy_file=None, rules=None, default_rule=None, use_conf=True):
+def init(policy_file=None, rules=None, default_rule=None, use_conf=True,
+         suppress_deprecation_warnings=False):
     """Init an Enforcer class.
 
        :param policy_file: Custom policy file to use, if none is specified,
@@ -55,6 +56,8 @@ def init(policy_file=None, rules=None, default_rule=None, use_conf=True):
        :param default_rule: Default rule to use, CONF.default_rule will
                             be used if none is specified.
        :param use_conf: Whether to load rules from config file.
+       :param suppress_deprecation_warnings: Whether to suppress the
+                                             deprecation warnings.
     """
     global _ENFORCER
     global saved_file_rules
@@ -64,6 +67,15 @@ def init(policy_file=None, rules=None, default_rule=None, use_conf=True):
                                     rules=rules,
                                     default_rule=default_rule,
                                     use_conf=use_conf)
+        # NOTE(gmann): Explictly disable the warnings for policies
+        # changing their default check_str. During policy-defaults-refresh
+        # work, all the policy defaults have been changed and warning for
+        # each policy started filling the logs limit for various tool.
+        # Once we move to new defaults only world then we can enable these
+        # warning again.
+        _ENFORCER.suppress_default_change_warnings = True
+        if suppress_deprecation_warnings:
+            _ENFORCER.suppress_deprecation_warnings = True
         register_rules(_ENFORCER)
         _ENFORCER.load_rules()
 

masakari/tests/unit/policy_fixture.py

@@ -13,10 +13,10 @@
 # under the License.
 
 import os
+import yaml
 
 import fixtures
 from oslo_policy import policy as oslo_policy
-from oslo_serialization import jsonutils
 
 import masakari.conf
 from masakari.conf import paths
@@ -46,11 +46,11 @@ class RealPolicyFixture(fixtures.Fixture):
     def setUp(self):
         super(RealPolicyFixture, self).setUp()
         # policy_file can be overridden by subclasses
-        self.policy_file = paths.state_path_def('etc/masakari/policy.json')
+        self.policy_file = paths.state_path_def('etc/masakari/policy.yaml')
         self._prepare_policy()
         CONF.set_override('policy_file', self.policy_file, group='oslo_policy')
         masakari.policy.reset()
-        masakari.policy.init()
+        masakari.policy.init(suppress_deprecation_warnings=True)
         self.addCleanup(masakari.policy.reset)
 
     def set_rules(self, rules, overwrite=True):
@@ -68,13 +68,13 @@ class PolicyFixture(RealPolicyFixture):
     """
     def _prepare_policy(self):
         self.policy_dir = self.useFixture(fixtures.TempDir())
-        self.policy_file = os.path.join(self.policy_dir.path, 'policy.json')
+        self.policy_file = os.path.join(self.policy_dir.path, 'policy.yaml')
 
         # load the fake_policy data and add the missing default rules.
-        policy_rules = jsonutils.loads(fake_policy.policy_data)
+        policy_rules = yaml.safe_load(fake_policy.policy_data)
 
         with open(self.policy_file, 'w') as f:
-            jsonutils.dump(policy_rules, f)
+            yaml.dump(policy_rules, f)
         CONF.set_override('policy_dirs', [], group='oslo_policy')
 
 
@@ -99,6 +99,6 @@ class RoleBasedPolicyFixture(RealPolicyFixture):
             policy[rule.name] = 'role:%s' % self.role
 
         self.policy_dir = self.useFixture(fixtures.TempDir())
-        self.policy_file = os.path.join(self.policy_dir.path, 'policy.json')
+        self.policy_file = os.path.join(self.policy_dir.path, 'policy.yaml')
         with open(self.policy_file, 'w') as f:
-            jsonutils.dump(policy, f)
+            yaml.dump(policy, f)

masakari/tests/unit/test_api_validation.py

@@ -18,7 +18,6 @@ import re
 
 import fixtures
 from jsonschema import exceptions as jsonschema_exc
-import sys
 
 from masakari.api import api_version_request as api_version
 from masakari.api import validation
@@ -269,10 +268,7 @@ class PatternPropertiesTestCase(APIValidationTestCase):
         self.check_validation_error(self.post, body={'0123456789a': 'bar'},
                                     expected_detail=details)
 
-        if sys.version[:3] in ['3.5', '3.6', '3.7']:
         detail = "expected string or bytes-like object"
-        else:
-            detail = "expected string or buffer"
         self.check_validation_error(self.post, body={None: 'bar'},
                                     expected_detail=detail)
 

masakari/tests/unit/test_policy.py

@@ -49,7 +49,7 @@ class PolicyFileTestCase(test.NoDBTestCase):
             # determine is_admin or not. As a side-effect, policy reset is
             # needed here to flush existing policy cache.
             policy.reset()
-            policy.init()
+            policy.init(suppress_deprecation_warnings=True)
             rule = oslo_policy.RuleDefault('example:test', "")
             policy._ENFORCER.register_defaults([rule])
 
@@ -84,7 +84,7 @@ class PolicyTestCase(test.NoDBTestCase):
                                     "role:ADMIN or role:sysadmin"),
         ]
         policy.reset()
-        policy.init()
+        policy.init(suppress_deprecation_warnings=True)
         # before a policy rule can be used, its default has to be registered.
         policy._ENFORCER.register_defaults(rules)
         self.context = context.RequestContext('fake', 'fake', roles=['member'])
@@ -158,7 +158,7 @@ class PolicyTestCase(test.NoDBTestCase):
 class IsAdminCheckTestCase(test.NoDBTestCase):
     def setUp(self):
         super(IsAdminCheckTestCase, self).setUp()
-        policy.init()
+        policy.init(suppress_deprecation_warnings=True)
 
     def test_init_true(self):
         check = policy.IsAdminCheck('is_admin', 'True')

requirements.txt

@@ -11,7 +11,7 @@ WebOb>=1.7.1 # MIT
 microversion-parse>=0.2.1 # Apache-2.0
 oslo.config>=5.2.0 # Apache-2.0
 oslo.context>=2.19.2 # Apache-2.0
-oslo.db>=4.27.0 # Apache-2.0
+oslo.db>=4.44.0 # Apache-2.0
 oslo.messaging>=5.29.0 # Apache-2.0
 oslo.i18n>=3.15.3 # Apache-2.0
 oslo.log>=3.36.0 # Apache-2.0

test-requirements.txt

@@ -7,7 +7,7 @@ hacking>=3.0.1,<3.1.0 # Apache-2.0
 coverage!=4.4,>=4.0 # Apache-2.0
 ddt>=1.0.1 # MIT
 pep8>=1.5.7
-psycopg2>=2.7 # LGPL/ZPL
+psycopg2>=2.8 # LGPL/ZPL
 PyMySQL>=0.7.6 # MIT License
 python-subunit>=1.0.0 # Apache-2.0/BSD
 openstacksdk>=0.35.0 # Apache-2.0