starlingx/utilities
Code Issues Proposed changes

Ceph-manager: Change timeout usage for ceph commands

Browse Source 
Ceph-manager is using cephclient to get information from ceph and
the command 'ceph restful list-keys --connect-timeout 15' is
timing out and crashing, generating core dumps.

Removed the '-connect-timeout' parameter and added 'timeout'
argument in check_output function.
This way no core dumps were seen.

Test Plan:
  PASS: Fresh install AIO-SX, stop ceph services, watch ceph-manager.log
        file looking for timed out commands and verify if there is any
        core dump from ceph.

Closes-bug: 1999985

Signed-off-by: Felipe Sanches Zanoni <Felipe.SanchesZanoni@windriver.com>
Change-Id: Ie119cacd2409de07ed700fd554df03d4187a252d
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
This commit is contained in:
Felipe Sanches Zanoni 2022-12-17 14:07:43 -05:00 committed by Erickson Silva de Oliveira
parent 0f534fed8c
commit 21236a3d4b
2 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
ceph/python-cephclient/python-cephclient/cephclient/client.py
View File

@ -69,12 +69,13 @@ class CephClient(object):
def _get_password(self): def _get_password(self):
try: try:
output = subprocess.check_output( output = subprocess.check_output(
('ceph restful list-keys ' 'ceph restful list-keys',
'--connect-timeout {}').format( timeout=CEPH_CLI_TIMEOUT_SEC,
CEPH_CLI_TIMEOUT_SEC),
shell=True) shell=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise exception.CephMonRestfulListKeysError(str(e)) raise exception.CephMonRestfulListKeysError(str(e))
except subprocess.TimeoutExpired as e:
raise exception.CephCliTimeoutExpired(str(e))
try: try:
keys = json.loads(output) keys = json.loads(output)
except (KeyError, ValueError): except (KeyError, ValueError):
@ -89,12 +90,13 @@ class CephClient(object):
while attempts <= CEPH_GET_SERVICE_RETRY_COUNT: while attempts <= CEPH_GET_SERVICE_RETRY_COUNT:
try: try:
output = subprocess.check_output( output = subprocess.check_output(
('ceph mgr services ' 'ceph mgr services',
'--connect-timeout {}').format( timeout=CEPH_CLI_TIMEOUT_SEC,
CEPH_CLI_TIMEOUT_SEC),
shell=True) shell=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise exception.CephMgrDumpError(str(e)) raise exception.CephMgrDumpError(str(e))
except subprocess.TimeoutExpired as e:
raise exception.CephCliTimeoutExpired(str(e))
try: try:
status = json.loads(output) status = json.loads(output)
if not status: if not status:
@ -115,12 +117,13 @@ class CephClient(object):
def _get_service_hostname(self): def _get_service_hostname(self):
try: try:
output = subprocess.check_output( output = subprocess.check_output(
('ceph mgr metadata ' 'ceph mgr metadata',
'--connect-timeout {}').format( timeout=CEPH_CLI_TIMEOUT_SEC,
CEPH_CLI_TIMEOUT_SEC),
shell=True) shell=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise exception.CephMgrDumpError(str(e)) raise exception.CephMgrDumpError(str(e))
except subprocess.TimeoutExpired as e:
raise exception.CephCliTimeoutExpired(str(e))
try: try:
status = json.loads(output) status = json.loads(output)
except (KeyError, ValueError): except (KeyError, ValueError):
@ -133,12 +136,11 @@ class CephClient(object):
try: try:
certificate = subprocess.check_output( certificate = subprocess.check_output(
('ceph config-key get ' ('ceph config-key get '
'--connect-timeout {} '
'mgr/restful/{}/crt').format( 'mgr/restful/{}/crt').format(
CEPH_CLI_TIMEOUT_SEC,
hostname), hostname),
timeout=CEPH_CLI_TIMEOUT_SEC,
shell=True) shell=True)
except subprocess.CalledProcessError: except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
return return
with tempfile.NamedTemporaryFile(delete=False) as self.cert_file: with tempfile.NamedTemporaryFile(delete=False) as self.cert_file:
self.cert_file.write(certificate) self.cert_file.write(certificate)

3
ceph/python-cephclient/python-cephclient/cephclient/exception.py
View File

@ -99,3 +99,6 @@ class CephClientNoSuchUser(CephClientException):
class CephClientIncorrectPassword(CephClientException): class CephClientIncorrectPassword(CephClientException):
message = ("Incorrect password for user '{user}'.") message = ("Incorrect password for user '{user}'.")
class CephCliTimeoutExpired(CephClientException):
message = "Timeout was reached while executing the command. {}"