openstack/swift
Code Issues Proposed changes

memcachering: change failed to yield log message

Browse Source 
Currently when the memcachering `_get_conns` method runs out of memcached
servers to try and so fails to yield anything we log a:

  All memcached servers error-limited

However, this error message isn't entirely accurate. It can also fail
because it failed to connect all it's memcached servers not just because
they're error limited.
You can disable error-limiting of memcached servers. So in this case
this error message is a red-herring.

Downstream we use a mcrouter client on each node which itself talks to a
bunch of memcache servers. Therefore in swift's memcachering client we
only configure the 1 mcrouter client as a single server in the ring.
Because of this we disable memcached error-limiting.
If the node gets too overloaded we've had timeouts talking to the local
mcrouter client. This fires off error-limitted log messages which can
confuse things.

Because it's possible to turn off error-limiting, the log line isn't
quite adequate anymore. So this patch changes it to:

  No more memcached servers to try

Change-Id: I97fb4f3ee2ac45831aae14a782b2c6dc73e82d85
This commit is contained in:
Matthew Oliver 2024-03-05 14:44:37 +11:00
parent 627448362a
commit 4135133a63
2 changed files with 7 additions and 7 deletions
swift/common
memcached.py
test/unit/common
test_memcached.py

2
swift/common/memcached.py

@ -375,7 +375,7 @@ class MemcacheRing(object):
action='connecting', sock=sock)
if not any_yielded:
self._log_error('ALL', cmd, 'connecting',
'All memcached servers error-limited')
'No more memcached servers to try')
def _return_conn(self, server, fp, sock):
"""Returns a server connection to the pool."""

12
test/unit/common/test_memcached.py

@ -655,7 +655,7 @@ class TestMemcached(unittest.TestCase):
'[Errno 32] Broken pipe',
'Error limiting server 1.2.3.4:11211',
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method set: All memcached servers error-limited',
'method set: No more memcached servers to try',
])
self.logger.clear()
@ -664,7 +664,7 @@ class TestMemcached(unittest.TestCase):
memcache_client.set('some_key', [1, 2, 3])
self.assertEqual(self.logger.get_lines_for_level('error'), [
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method set: All memcached servers error-limited',
'method set: No more memcached servers to try',
] * 12)
self.logger.clear()
@ -672,7 +672,7 @@ class TestMemcached(unittest.TestCase):
self.assertIsNone(memcache_client.get('some_key'))
self.assertEqual(self.logger.get_lines_for_level('error'), [
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method get: All memcached servers error-limited',
'method get: No more memcached servers to try',
])
def test_error_disabled(self):
@ -790,7 +790,7 @@ class TestMemcached(unittest.TestCase):
'[Errno 32] Broken pipe',
'Error limiting server 1.2.3.5:11211',
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method set: All memcached servers error-limited',
'method set: No more memcached servers to try',
])
# with default error_limit_time of 60, one call per 6 secs, error limit
@ -815,7 +815,7 @@ class TestMemcached(unittest.TestCase):
'[Errno 32] Broken pipe',
'Error limiting server 1.2.3.5:11211',
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method set: All memcached servers error-limited'])
'method set: No more memcached servers to try'])
# with error_limit_time of 70, one call per 6 secs, error_limit_count
# of 11, 13th call triggers error limit
@ -830,7 +830,7 @@ class TestMemcached(unittest.TestCase):
'[Errno 32] Broken pipe',
'Error limiting server 1.2.3.5:11211',
'Error connecting to memcached: ALL: with key_prefix some_key, '
'method set: All memcached servers error-limited'])
'method set: No more memcached servers to try'])
def test_delete(self):
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],