diff --git a/neutron/agent/common/ovs_lib.py b/neutron/agent/common/ovs_lib.py index c837b2f3e8b..d22b0897a3e 100644 --- a/neutron/agent/common/ovs_lib.py +++ b/neutron/agent/common/ovs_lib.py @@ -23,8 +23,8 @@ from neutron_lib import exceptions from oslo_config import cfg from oslo_log import log as logging from oslo_utils import excutils -import retrying import six +import tenacity from neutron._i18n import _, _LE, _LI, _LW from neutron.agent.common import utils @@ -74,12 +74,12 @@ def _ofport_retry(fn): @six.wraps(fn) def wrapped(*args, **kwargs): self = args[0] - new_fn = retrying.retry( - retry_on_result=_ofport_result_pending, - stop_max_delay=self.vsctl_timeout * 1000, - wait_exponential_multiplier=10, - wait_exponential_max=1000, - retry_on_exception=lambda _: False)(fn) + new_fn = tenacity.retry( + reraise=True, + retry=tenacity.retry_if_result(_ofport_result_pending), + wait=tenacity.wait_exponential(multiplier=0.01, max=1), + stop=tenacity.stop_after_delay( + self.vsctl_timeout))(fn) return new_fn(*args, **kwargs) return wrapped @@ -275,7 +275,7 @@ class OVSBridge(BaseOVS): ofport = INVALID_OFPORT try: ofport = self._get_port_ofport(port_name) - except retrying.RetryError: + except tenacity.RetryError: LOG.exception(_LE("Timed out retrieving ofport on port %s."), port_name) return ofport diff --git a/neutron/agent/ovsdb/native/connection.py b/neutron/agent/ovsdb/native/connection.py index ebc7529b933..cd9494b5ba6 100644 --- a/neutron/agent/ovsdb/native/connection.py +++ b/neutron/agent/ovsdb/native/connection.py @@ -18,8 +18,8 @@ import traceback from ovs.db import idl from ovs import poller -import retrying from six.moves import queue as Queue +import tenacity from neutron.agent.ovsdb.native import helpers from neutron.agent.ovsdb.native import idlutils @@ -96,8 +96,9 @@ class Connection(object): helpers.enable_connection_uri(self.connection) # There is a small window for a race, so retry up to a second - @retrying.retry(wait_exponential_multiplier=10, - stop_max_delay=1000) + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01), + stop=tenacity.stop_after_delay(1), + reraise=True) def do_get_schema_helper(): return idlutils.get_schema_helper(self.connection, self.schema_name) diff --git a/neutron/tests/unit/agent/common/test_ovs_lib.py b/neutron/tests/unit/agent/common/test_ovs_lib.py index 7edf86c2589..dff216c841d 100644 --- a/neutron/tests/unit/agent/common/test_ovs_lib.py +++ b/neutron/tests/unit/agent/common/test_ovs_lib.py @@ -18,6 +18,7 @@ import mock from neutron_lib import exceptions from oslo_serialization import jsonutils from oslo_utils import uuidutils +import tenacity import testtools from neutron.agent.common import config @@ -805,6 +806,20 @@ class OVS_Lib_Test(base.BaseTestCase): 'tap99id', data, extra_calls_and_values=extra_calls_and_values) self._assert_vif_port(vif_port, ofport=1337, mac="de:ad:be:ef:13:37") + def test_get_port_ofport_retry(self): + with mock.patch.object( + self.br, 'db_get_val', + side_effect=[[], [], [], [], 1]): + self.assertEqual(1, self.br._get_port_ofport('1')) + + def test_get_port_ofport_retry_fails(self): + # after 16 calls the retry will timeout and raise + with mock.patch.object( + self.br, 'db_get_val', + side_effect=[[] for _ in range(16)]): + self.assertRaises(tenacity.RetryError, + self.br._get_port_ofport, '1') + class TestDeferredOVSBridge(base.BaseTestCase): diff --git a/neutron/tests/unit/agent/ovsdb/native/test_connection.py b/neutron/tests/unit/agent/ovsdb/native/test_connection.py index f60ea070bab..0b0ea091a29 100644 --- a/neutron/tests/unit/agent/ovsdb/native/test_connection.py +++ b/neutron/tests/unit/agent/ovsdb/native/test_connection.py @@ -82,3 +82,23 @@ class TestOVSNativeConnection(base.BaseTestCase): idl_instance = idl_class.return_value self.connection.start() self.assertEqual(idl_instance, self.connection.idl) + + @mock.patch.object(connection, 'threading') + @mock.patch.object(connection.idlutils, 'wait_for_change') + @mock.patch.object(connection, 'idl') + @mock.patch.object(connection.helpers, 'enable_connection_uri') + @mock.patch.object(connection.idlutils, 'get_schema_helper') + def test_do_get_schema_helper_retry(self, mock_get_schema_helper, + mock_enable_conn, + mock_idl, + mock_wait_for_change, + mock_threading): + mock_helper = mock.Mock() + # raise until 3rd retry attempt + mock_get_schema_helper.side_effect = [Exception(), Exception(), + mock_helper] + conn = connection.Connection( + mock.Mock(), mock.Mock(), mock.Mock()) + conn.start() + self.assertEqual(3, len(mock_get_schema_helper.mock_calls)) + mock_helper.register_all.assert_called_once_with() diff --git a/requirements.txt b/requirements.txt index f02c1642af6..62f88dd2f16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ netaddr!=0.7.16,>=0.7.13 # BSD netifaces>=0.10.4 # MIT neutron-lib>=0.4.0 # Apache-2.0 python-neutronclient>=5.1.0 # Apache-2.0 -retrying!=1.3.0,>=1.2.3 # Apache-2.0 +tenacity>=3.1.1 # Apache-2.0 ryu!=4.1,!=4.2,!=4.2.1,!=4.4,>=3.30 # Apache-2.0 SQLAlchemy<1.1.0,>=1.0.10 # MIT WebOb>=1.6.0 # MIT