Reconnect on connection lost in heartbeat thread

During the log refactoring of the big heartbeat patch,
one line have been squashed (line 936):

https://review.openstack.org/#/c/146047/29..30/oslo_messaging/_drivers/impl_rabbit.py

This change reintroduces it, and improve the test to cover it.

Change-Id: I5e7162f37527580c14a809f9945114dc81451c1a

oslo_messaging/_drivers/impl_rabbit.py

@@ -951,6 +951,7 @@ class Connection(object):
                     except recoverable_errors as exc:
                         LOG.info(_LI("A recoverable connection/channel error "
                                      "occurred, trying to reconnect: %s"), exc)
+                        self.ensure_connection()
                 except Exception:
                     LOG.exception(_LE("Unexpected error during heartbeart "
                                       "thread processing, retrying..."))

oslo_messaging/tests/drivers/test_impl_rabbit.py

@@ -62,7 +62,10 @@ class TestHeartbeat(test_utils.BaseTestCase):
     @mock.patch('kombu.connection.Connection.heartbeat_check')
     @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.'
                 '_heartbeat_supported_and_enabled', return_value=True)
-    def _do_test_heartbeat_sent(self, fake_heartbeat_support, fake_heartbeat,
+    @mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.'
+                'ensure_connection')
+    def _do_test_heartbeat_sent(self, fake_ensure_connection,
+                                fake_heartbeat_support, fake_heartbeat,
                                 fake_logger, heartbeat_side_effect=None,
                                 info=None):
 
@@ -79,6 +82,7 @@ class TestHeartbeat(test_utils.BaseTestCase):
                                                  'kombu+memory:////')
         self.addCleanup(transport.cleanup)
         conn = transport._driver._get_connection()
+        conn.ensure(method=lambda: True)
         event.wait()
         conn._heartbeat_stop()
 
@@ -86,8 +90,10 @@ class TestHeartbeat(test_utils.BaseTestCase):
         self.assertLess(0, fake_heartbeat.call_count)
 
         if not heartbeat_side_effect:
+            self.assertEqual(1, fake_ensure_connection.call_count)
             self.assertEqual(2, fake_logger.info.call_count)
         else:
+            self.assertEqual(2, fake_ensure_connection.call_count)
             self.assertEqual(3, fake_logger.info.call_count)
             self.assertIn(mock.call(info, mock.ANY),
                           fake_logger.info.mock_calls)