Merge "Catch broad exception in methods used in FixedIntervalLoopingCall"

neutron/db/agentschedulers_db.py

@@ -270,40 +270,47 @@ class DhcpAgentSchedulerDbMixin(dhcpagentscheduler
                    agents_db.Agent.admin_state_up))
         dhcp_notifier = self.agent_notifiers.get(constants.AGENT_TYPE_DHCP)
 
-        for binding in self._filter_bindings(context, down_bindings):
+        try:
-            LOG.warn(_LW("Removing network %(network)s from agent %(agent)s "
+            for binding in self._filter_bindings(context, down_bindings):
-                         "because the agent did not report to the server in "
+                LOG.warn(_LW("Removing network %(network)s from agent "
-                         "the last %(dead_time)s seconds."),
+                             "%(agent)s because the agent did not report "
-                     {'network': binding.network_id,
+                             "to the server in the last %(dead_time)s "
-                      'agent': binding.dhcp_agent_id,
+                             "seconds."),
-                      'dead_time': agent_dead_limit})
+                         {'network': binding.network_id,
-            # save binding object to avoid ObjectDeletedError
+                          'agent': binding.dhcp_agent_id,
-            # in case binding is concurrently deleted from the DB
+                          'dead_time': agent_dead_limit})
-            saved_binding = {'net': binding.network_id,
+                # save binding object to avoid ObjectDeletedError
-                             'agent': binding.dhcp_agent_id}
+                # in case binding is concurrently deleted from the DB
-            try:
+                saved_binding = {'net': binding.network_id,
-                # do not notify agent if it considered dead
+                                 'agent': binding.dhcp_agent_id}
-                # so when it is restarted it won't see network delete
+                try:
-                # notifications on its queue
+                    # do not notify agent if it considered dead
-                self.remove_network_from_dhcp_agent(context,
+                    # so when it is restarted it won't see network delete
-                                                    binding.dhcp_agent_id,
+                    # notifications on its queue
-                                                    binding.network_id,
+                    self.remove_network_from_dhcp_agent(context,
-                                                    notify=False)
+                                                        binding.dhcp_agent_id,
-            except dhcpagentscheduler.NetworkNotHostedByDhcpAgent:
+                                                        binding.network_id,
-                # measures against concurrent operation
+                                                        notify=False)
-                LOG.debug("Network %(net)s already removed from DHCP agent "
+                except dhcpagentscheduler.NetworkNotHostedByDhcpAgent:
-                          "%(agent)s",
+                    # measures against concurrent operation
-                          saved_binding)
+                    LOG.debug("Network %(net)s already removed from DHCP "
-                # still continue and allow concurrent scheduling attempt
+                              "agent %(agent)s",
-            except Exception:
-                LOG.exception(_LE("Unexpected exception occurred while "
-                                  "removing network %(net)s from agent "
-                                  "%(agent)s"),
                               saved_binding)
+                    # still continue and allow concurrent scheduling attempt
+                except Exception:
+                    LOG.exception(_LE("Unexpected exception occurred while "
+                                      "removing network %(net)s from agent "
+                                      "%(agent)s"),
+                                  saved_binding)
 
-            if cfg.CONF.network_auto_schedule:
+                if cfg.CONF.network_auto_schedule:
-                self._schedule_network(
+                    self._schedule_network(
-                    context, saved_binding['net'], dhcp_notifier)
+                        context, saved_binding['net'], dhcp_notifier)
+        except Exception:
+            # we want to be thorough and catch whatever is raised
+            # to avoid loop abortion
+            LOG.exception(_LE("Exception encountered during network "
+                              "rescheduling"))
 
     def get_dhcp_agents_hosting_networks(
             self, context, network_ids, active=None, admin_state_up=None):

neutron/db/l3_agentschedulers_db.py

@@ -116,9 +116,9 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
                     # so one broken one doesn't stop the iteration.
                     LOG.exception(_LE("Failed to reschedule router %s"),
                                   binding.router_id)
-        except db_exc.DBError:
+        except Exception:
-            # Catch DB errors here so a transient DB connectivity issue
+            # we want to be thorough and catch whatever is raised
-            # doesn't stop the loopingcall.
+            # to avoid loop abortion
             LOG.exception(_LE("Exception encountered during router "
                               "rescheduling."))
 

neutron/tests/unit/plugins/openvswitch/test_agent_scheduler.py

@@ -674,17 +674,14 @@ class OvsAgentSchedulerTestCase(OvsAgentSchedulerTestCaseBase):
                     db_exc.DBError(), oslo_messaging.RemoteError(),
                     l3agentscheduler.RouterReschedulingFailed(router_id='f',
                                                               agent_id='f'),
-                    ValueError('this raises')
+                    ValueError('this raises'),
+                    Exception()
                 ]).start()
-            # these first three should not raise any errors
             self._take_down_agent_and_run_reschedule(L3_HOSTA)  # DBError
             self._take_down_agent_and_run_reschedule(L3_HOSTA)  # RemoteError
             self._take_down_agent_and_run_reschedule(L3_HOSTA)  # schedule err
-
+            self._take_down_agent_and_run_reschedule(L3_HOSTA)  # Value error
-            # ValueError is not caught so it should raise
+            self._take_down_agent_and_run_reschedule(L3_HOSTA)  # Exception
-            self.assertRaises(ValueError,
-                              self._take_down_agent_and_run_reschedule,
-                              L3_HOSTA)
 
     def test_router_rescheduler_iterates_after_reschedule_failure(self):
         plugin = manager.NeutronManager.get_service_plugins().get(

neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py

@@ -248,6 +248,15 @@ class TestNetworksFailover(TestDhcpSchedulerBaseTestCase,
             self.assertIn('foo3', res_ids)
             self.assertIn('foo4', res_ids)
 
+    def test_reschedule_network_from_down_agent_failed_on_unexpected(self):
+        agents = self._create_and_set_agents_down(['host-a'], 1)
+        self._test_schedule_bind_network([agents[0]], self.network_id)
+        with mock.patch.object(
+            self, '_filter_bindings',
+            side_effect=Exception()):
+            # just make sure that no exception is raised
+            self.remove_networks_from_down_agents()
+
 
 class DHCPAgentWeightSchedulerTestCase(TestDhcpSchedulerBaseTestCase):
     """Unit test scenarios for WeightScheduler.schedule."""