From e55ba3fb7d033ab948ae6d8af5d29c438b025c7c Mon Sep 17 00:00:00 2001 From: ppeng Date: Mon, 12 May 2025 14:04:50 -0400 Subject: [PATCH] Add ceph -s keywords - Add is_ceph_healthy() - Add wait_for_ceph_health_status() - Add get_ceph_osd_count() - Add get_ceph_mon_count() - Add unit_tests ceph_s_table_parser_test.py Change-Id: Ibda00d122a704d8741deb15be0b53afcf2d6654d Signed-off-by: ppeng --- keywords/ceph/ceph_status_keywords.py | 52 +++++ .../ceph/ceph_status_section_table_parser.py | 59 ++++++ .../ceph/object/ceph_status_cluster_object.py | 26 +++ .../ceph/object/ceph_status_cluster_output.py | 51 +++++ .../ceph/object/ceph_status_data_object.py | 56 ++++++ .../ceph/object/ceph_status_data_output.py | 56 ++++++ keywords/ceph/object/ceph_status_io_object.py | 16 ++ keywords/ceph/object/ceph_status_io_output.py | 40 ++++ keywords/ceph/object/ceph_status_output.py | 186 ++++++++++++++++++ .../object/ceph_status_services_object.py | 46 +++++ .../object/ceph_status_services_output.py | 52 +++++ .../ceph/ceph_status_table_parser_test.py | 84 ++++++++ 12 files changed, 724 insertions(+) create mode 100644 keywords/ceph/ceph_status_keywords.py create mode 100644 keywords/ceph/ceph_status_section_table_parser.py create mode 100644 keywords/ceph/object/ceph_status_cluster_object.py create mode 100644 keywords/ceph/object/ceph_status_cluster_output.py create mode 100644 keywords/ceph/object/ceph_status_data_object.py create mode 100644 keywords/ceph/object/ceph_status_data_output.py create mode 100644 keywords/ceph/object/ceph_status_io_object.py create mode 100644 keywords/ceph/object/ceph_status_io_output.py create mode 100644 keywords/ceph/object/ceph_status_output.py create mode 100644 keywords/ceph/object/ceph_status_services_object.py create mode 100644 keywords/ceph/object/ceph_status_services_output.py create mode 100644 unit_tests/parser/ceph/ceph_status_table_parser_test.py diff --git a/keywords/ceph/ceph_status_keywords.py b/keywords/ceph/ceph_status_keywords.py new file mode 100644 index 00000000..6eea31d4 --- /dev/null +++ b/keywords/ceph/ceph_status_keywords.py @@ -0,0 +1,52 @@ +from framework.ssh.ssh_connection import SSHConnection +from framework.validation.validation import validate_equals_with_retry +from keywords.base_keyword import BaseKeyword +from keywords.ceph.object.ceph_status_output import CephStatusOutput + + +class CephStatusKeywords(BaseKeyword): + """ + Class for ceph -s Keywords + + """ + + def __init__(self, ssh_connection: SSHConnection): + self.ssh_connection = ssh_connection + + def ceph_status(self) -> CephStatusOutput: + """ + Run ceph -s command + + Args: None + + Returns: CephStatusOutput + + """ + output = self.ssh_connection.send("ceph -s") + self.validate_success_return_code(self.ssh_connection) + ceph_status_output = CephStatusOutput(output) + return ceph_status_output + + def wait_for_ceph_health_status(self, expect_health_status: bool = None, timeout: int = 1800) -> bool: + """ + Waits timeout amount of time for ceph to be in the given status + + Args: + expect_health_status (bool): Ture (HEALTH_OK) or False (HEALTH_WARN) + timeout (int): the timeout in secs + + Returns: + bool: True: ceph health status match expect status + False: ceph health status not match expect status + + """ + if expect_health_status not in (True, False): + raise ValueError(f"expect_health_status:{expect_health_status} is not valid.") + + def get_ceph_health_status(): + output = self.ssh_connection.send("ceph -s") + ceph_status_output = CephStatusOutput(output) + return ceph_status_output.is_ceph_healthy() + + msg = f"Current ceph health status should match expected status:{expect_health_status}" + validate_equals_with_retry(get_ceph_health_status, expect_health_status, msg, timeout=timeout) diff --git a/keywords/ceph/ceph_status_section_table_parser.py b/keywords/ceph/ceph_status_section_table_parser.py new file mode 100644 index 00000000..03cc154c --- /dev/null +++ b/keywords/ceph/ceph_status_section_table_parser.py @@ -0,0 +1,59 @@ +import re + +from framework.exceptions.keyword_exception import KeywordException + + +class CephStatusSectionTableParser: + """ + Class for ceph -s section table parsing + + Example: + ["health: HEALTH_WARN", + "2 MDSs report slow metadata IOs", + "Reduced data availability: 48 pgs inactive", + "Degraded data redundancy: 66/196 objects degraded (33.673%), 11 pgs degraded, 48 pgs undersized"] + + OR + + ["mon: 3 daemons, quorum a,b,c (age 2w)", + "mgr: b(active, since 46s), standbys: c, a", + "mds: 1/1 daemons up, 1 hot standby", + "osd: 5 osds: 5 up (since 2w), 5 in (since 2w)"] + """ + + def __init__(self, ceph_status_section_output: list[str]): + """ + Constructor + + Args: + ceph_status_section_output (list[str]): a list of strings representing one section output of 'ceph -s' command + """ + self.ceph_status_section_output = ceph_status_section_output + + def get_output_values_dict(self) -> {}: + """ + Getter for output values dict + + Returns: + {}: the output values dict + + """ + output_values_dict = {} + for row in self.ceph_status_section_output: + # Only health section has extra lines + if "health" in output_values_dict: + output_values_dict["health"] += "; " + row + continue + # splits the string at the first colon followed by any amount of whitespace. + values = re.split(r":\s*", row, maxsplit=1) + if len(values) == 2: + key, value = values + output_values_dict[key] = value + else: + # just a newline -- continue + if not values or len(values) == 1: + continue + else: + raise KeywordException(f"Line with values: {row} was not in the expected format") + + return output_values_dict diff --git a/keywords/ceph/object/ceph_status_cluster_object.py b/keywords/ceph/object/ceph_status_cluster_object.py new file mode 100644 index 00000000..d916f30f --- /dev/null +++ b/keywords/ceph/object/ceph_status_cluster_object.py @@ -0,0 +1,26 @@ +class CephClusterObject: + """ + Object to hold the values of Ceph Cluster Object + """ + + def __init__(self): + self.id: str = "" + self.health: str = "" + + def get_id(self) -> str: + """ + Getter for id + + Returns: id + + """ + return self.id + + def get_health(self) -> str: + """ + Getter for ceph health status + + Returns: health + + """ + return self.health diff --git a/keywords/ceph/object/ceph_status_cluster_output.py b/keywords/ceph/object/ceph_status_cluster_output.py new file mode 100644 index 00000000..7fd4d58d --- /dev/null +++ b/keywords/ceph/object/ceph_status_cluster_output.py @@ -0,0 +1,51 @@ +from keywords.ceph.ceph_status_section_table_parser import CephStatusSectionTableParser +from keywords.ceph.object.ceph_status_cluster_object import CephClusterObject + + +class CephClusterOutput: + """ + This class parses the output of Ceph Cluster + + Example: + id: 8abb43ce-6775-4a1a-99c4-12f37101410e + health: HEALTH_WARN + 2 MDSs report slow metadata IOs + Reduced data availability: 48 pgs inactive + Degraded data redundancy: 66/196 objects degraded (33.673%), 11 pgs degraded, 48 pgs undersized + + OR + + id: 8abb43ce-6775-4a1a-99c4-12f37101410e + health: HEALTH_OK + + """ + + def __init__(self, ceph_cluster_output: list[str]): + """ + Constructor + + Create an internal CephClusterObject. + + Args: + ceph_cluster_output (list[str]): a list of strings representing the cluster output + + """ + ceph_table_parser = CephStatusSectionTableParser(ceph_cluster_output) + output_values = ceph_table_parser.get_output_values_dict() + self.ceph_cluster_object = CephClusterObject() + + if "id" in output_values: + self.ceph_cluster_object.id = output_values["id"] + + if "health" in output_values: + self.ceph_cluster_object.health = output_values["health"] + + def get_ceph_cluster_object(self) -> CephClusterObject: + """ + Getter for CephClusterObject object. + + Returns (CephClusterObject): + A CephClusterObject + + """ + return self.ceph_cluster_object diff --git a/keywords/ceph/object/ceph_status_data_object.py b/keywords/ceph/object/ceph_status_data_object.py new file mode 100644 index 00000000..ef65c519 --- /dev/null +++ b/keywords/ceph/object/ceph_status_data_object.py @@ -0,0 +1,56 @@ +class CephDataObject: + """ + Object to hold the values of Ceph Data Object + """ + + def __init__(self): + self.volumes: str = "" + self.pools: str = "" + self.objects: str = "" + self.usage: str = "" + self.pgs: str = "" + + def get_volumes(self) -> str: + """ + Getter for volumes + + Returns: volumes + + """ + return self.volumes + + def get_pools(self) -> str: + """ + Getter for pools + + Returns: pools + + """ + return self.pools + + def get_objects(self) -> str: + """ + Getter for objects + + Returns: objects + + """ + return self.objects + + def get_usage(self) -> str: + """ + Getter for usage + + Returns: usage + + """ + return self.usage + + def get_pgs(self) -> str: + """ + Getter for pgs + + Returns: pgs + + """ + return self.pgs diff --git a/keywords/ceph/object/ceph_status_data_output.py b/keywords/ceph/object/ceph_status_data_output.py new file mode 100644 index 00000000..dae92426 --- /dev/null +++ b/keywords/ceph/object/ceph_status_data_output.py @@ -0,0 +1,56 @@ +from keywords.ceph.ceph_status_section_table_parser import CephStatusSectionTableParser +from keywords.ceph.object.ceph_status_data_object import CephDataObject + + +class CephDataOutput: + """ + This class parses the output of Ceph Data + + Example: + data: + volumes: 1/1 healthy + pools: 4 pools, 112 pgs + objects: 26 objects, 6.6 MiB + usage: 400 MiB used, 2.2 TiB / 2.2 TiB avail + pgs: 112 active+clean + + """ + + def __init__(self, ceph_data_output: list[str]): + """ + Constructor. + + Create an internal DataObject. + + Args: + ceph_data_output (list[str]): a list of strings representing the data output + + """ + ceph_table_parser = CephStatusSectionTableParser(ceph_data_output) + output_values = ceph_table_parser.get_output_values_dict() + self.ceph_data_object = CephDataObject() + + if "volumes" in output_values: + self.ceph_data_object.volumes = output_values["volumes"] + + if "pools" in output_values: + self.ceph_data_object.pools = output_values["pools"] + + if "objects" in output_values: + self.ceph_data_object.objects = output_values["objects"] + + if "usage" in output_values: + self.ceph_data_object.usage = output_values["usage"] + + if "pgs" in output_values: + self.ceph_data_object.pgs = output_values["pgs"] + + def get_ceph_data_object(self) -> CephDataObject: + """ + Getter for CephDataObject object. + + Returns (CephDataObject): + A CephDataObject + + """ + return self.ceph_data_object diff --git a/keywords/ceph/object/ceph_status_io_object.py b/keywords/ceph/object/ceph_status_io_object.py new file mode 100644 index 00000000..320f64a6 --- /dev/null +++ b/keywords/ceph/object/ceph_status_io_object.py @@ -0,0 +1,16 @@ +class CephIOObject: + """ + Object to hold the values of Ceph IO Object + """ + + def __init__(self): + self.client: str = "" + + def get_client(self) -> str: + """ + Getter for client + + Returns: client + + """ + return self.client diff --git a/keywords/ceph/object/ceph_status_io_output.py b/keywords/ceph/object/ceph_status_io_output.py new file mode 100644 index 00000000..c50b4c4f --- /dev/null +++ b/keywords/ceph/object/ceph_status_io_output.py @@ -0,0 +1,40 @@ +from keywords.ceph.ceph_status_section_table_parser import CephStatusSectionTableParser +from keywords.ceph.object.ceph_status_io_object import CephIOObject + + +class CephIOOutput: + """ + This class parses the output of Ceph IO + + Example: + io: + client: 853 B/s rd, 1 op/s rd, 0 op/s wr + + """ + + def __init__(self, ceph_io_output: list[str]): + """ + Constructor. + + Create an internal CephIOObject. + + Args: + ceph_io_output (list[str]): a list of strings representing the io output + + """ + ceph_table_parser = CephStatusSectionTableParser(ceph_io_output) + output_values = ceph_table_parser.get_output_values_dict() + self.ceph_io_object = CephIOObject() + + if "client" in output_values: + self.ceph_io_object.client = output_values["client"] + + def get_ceph_io_object(self) -> CephIOObject: + """ + Getter for CephIOObject object. + + Returns (CephIOObject): + A CephIOObject + + """ + return self.ceph_io_object diff --git a/keywords/ceph/object/ceph_status_output.py b/keywords/ceph/object/ceph_status_output.py new file mode 100644 index 00000000..d2b323aa --- /dev/null +++ b/keywords/ceph/object/ceph_status_output.py @@ -0,0 +1,186 @@ +import re + +from keywords.ceph.object.ceph_status_cluster_output import CephClusterOutput +from keywords.ceph.object.ceph_status_data_output import CephDataOutput +from keywords.ceph.object.ceph_status_io_output import CephIOOutput +from keywords.ceph.object.ceph_status_services_output import CephServicesOutput + + +class CephStatusOutput: + """ + This class parses the output of command ceph -s + + Example: + cluster: + id: 8abb43ce-6775-4a1a-99c4-12f37101410e + health: HEALTH_OK + + services: + mon: 3 daemons, quorum a,b,c (age 3w) + mgr: b(active, since 24h), standbys: c, a + mds: 1/1 daemons up, 1 hot standby + osd: 5 osds: 5 up (since 2w), 5 in (since 2w) + + data: + volumes: 1/1 healthy + pools: 4 pools, 112 pgs + objects: 26 objects, 6.6 MiB + usage: 401 MiB used, 2.2 TiB / 2.2 TiB avail + pgs: 112 active+clean + + io: + client: 1.2 KiB/s rd, 2 op/s rd, 0 op/s wr + + + """ + + def __init__(self, ceph_s_output: list[str]): + """ + Create a list of ceph objects for the given output + + Args: + ceph_s_output (list[str]): a list of strings representing the output of the ceph -s command + + """ + self.ceph_cluster_output: CephClusterOutput = None + self.ceph_services_output: CephServicesOutput = None + self.ceph_data_output: CephDataOutput = None + self.ceph_io_output: CephIOOutput = None + + section_name_list = ["cluster:", "services:", "data:", "io:"] + section_object = "" + body_str = [] + for line in ceph_s_output: + # remove front extra space of a line + line = line.lstrip() + # skip empty lines + if line.strip() == "": + continue + # get section name + elif any(section in line for section in section_name_list): + # We have completed a section content, now create it. + if section_object: + self.create_section_object(section_object, body_str) + # We are starting a new section. + # remove : and \n + section_object = line.replace(":", "").replace("\n", "") + body_str = [] + else: + # This line is part of the current section + # remove end '\n' + line = line.rstrip("\n") + body_str.append(line) + + # Create the last section + self.create_section_object(section_object, body_str) + + def create_section_object(self, section_object: str, body_str: list[str]): + """ + Creates the ceph section object + + Args: + section_object (str): the object to be created + body_str (list[str]): the body of the section + + """ + if "cluster" in section_object: + self.ceph_cluster_output = CephClusterOutput(body_str) + + if "services" in section_object: + self.ceph_services_output = CephServicesOutput(body_str) + + if "data" in section_object: + self.ceph_data_output = CephDataOutput(body_str) + + if "io" in section_object: + self.ceph_io_output = CephIOOutput(body_str) + + def get_ceph_cluster_output(self) -> CephClusterOutput: + """ + Getter for the ceph cluster output + + Returns: + CephClusterOutput: a CephClusterOutput object + + """ + return self.ceph_cluster_output + + def get_ceph_services_output(self) -> CephServicesOutput: + """ + Getter for the services output + + Returns: + CephServicesOutput: a CephServicesOutput object + + """ + return self.ceph_services_output + + def get_ceph_data_output(self) -> CephDataOutput: + """ + Getter for the data output + + Returns: + CephDataOutput: a CephDataOutput object + + """ + return self.ceph_data_output + + def get_ceph_io_output(self) -> CephIOOutput: + """ + Getter for the io output + + Returns: + CephIOOutput: a CephIOOutput object + + """ + return self.ceph_io_output + + def is_ceph_healthy(self) -> bool: + """ + Check whether ceph is healthy + + Args: None + + Returns: + bool: + If ceph health is ok, return True + If ceph health is not ok, return False + + """ + ceph_health_status_msg = self.ceph_cluster_output.get_ceph_cluster_object().get_health() + if "HEALTH_OK" in ceph_health_status_msg: + return True + elif "HEALTH_WARN" in ceph_health_status_msg: + return False + else: + raise ValueError("Ceph health status msg should content either HEALTH_OK or HEALTH_WARN") + + def get_ceph_osd_count(self) -> int: + """ + Get osd number + + Args: None + + Returns (int): + osd_number + + """ + osd_msg = self.ceph_services_output.get_ceph_services_object().get_osd() + match = re.search(r"\d+", osd_msg) + osd_number = int(match.group()) + return osd_number + + def get_ceph_mon_count(self) -> int: + """ + Get mons number + + Args: None + + Returns (int): + mons_number + + """ + mons_msg = self.ceph_services_output.get_ceph_services_object().get_mon() + match = re.search(r"\d+", mons_msg) + mons_number = int(match.group()) + return mons_number diff --git a/keywords/ceph/object/ceph_status_services_object.py b/keywords/ceph/object/ceph_status_services_object.py new file mode 100644 index 00000000..40b6bdfd --- /dev/null +++ b/keywords/ceph/object/ceph_status_services_object.py @@ -0,0 +1,46 @@ +class CephServicesObject: + """ + Object to hold the values of Ceph Services Object + """ + + def __init__(self): + self.mon: str = "" + self.mgr: str = "" + self.mds: str = "" + self.osd: str = "" + + def get_mon(self) -> str: + """ + Getter for mon + + Returns: mon + + """ + return self.mon + + def get_mgr(self) -> str: + """ + Getter for mgr + + Returns: mgr + + """ + return self.mgr + + def get_mds(self) -> str: + """ + Getter for mds + + Returns: mds + + """ + return self.mds + + def get_osd(self) -> str: + """ + Getter for osd + + Returns: osd + + """ + return self.osd diff --git a/keywords/ceph/object/ceph_status_services_output.py b/keywords/ceph/object/ceph_status_services_output.py new file mode 100644 index 00000000..2179996c --- /dev/null +++ b/keywords/ceph/object/ceph_status_services_output.py @@ -0,0 +1,52 @@ +from keywords.ceph.ceph_status_section_table_parser import CephStatusSectionTableParser +from keywords.ceph.object.ceph_status_services_object import CephServicesObject + + +class CephServicesOutput: + """ + This class parses the output of Services + + Example: + services: + mon: 3 daemons, quorum a,b,c (age 3w) + mgr: b(active, since 19h), standbys: c, a + mds: 1/1 daemons up, 1 hot standby + osd: 5 osds: 5 up (since 2w), 5 in (since 2w) + + """ + + def __init__(self, ceph_services_output: list[str]): + """ + Constructor. + + Create an internal CephServicesObject. + + Args: + ceph_services_output (list[str]): a list of strings representing the services output + + """ + ceph_table_parser = CephStatusSectionTableParser(ceph_services_output) + output_values = ceph_table_parser.get_output_values_dict() + self.ceph_services_object = CephServicesObject() + + if "mon" in output_values: + self.ceph_services_object.mon = output_values["mon"] + + if "mgr" in output_values: + self.ceph_services_object.mgr = output_values["mgr"] + + if "mds" in output_values: + self.ceph_services_object.mds = output_values["mds"] + + if "osd" in output_values: + self.ceph_services_object.osd = output_values["osd"] + + def get_ceph_services_object(self) -> CephServicesObject: + """ + Getter for CephServicesObject object. + + Returns (CephServicesObject): + A CephServicesObject + + """ + return self.ceph_services_object diff --git a/unit_tests/parser/ceph/ceph_status_table_parser_test.py b/unit_tests/parser/ceph/ceph_status_table_parser_test.py new file mode 100644 index 00000000..8bb5051e --- /dev/null +++ b/unit_tests/parser/ceph/ceph_status_table_parser_test.py @@ -0,0 +1,84 @@ +from keywords.ceph.object.ceph_status_output import CephStatusOutput + +# fmt: off +ceph_s_health_warning = [ + " cluster:\n", + " id: 001ab3d8-1f02-4294-b594-e2b216b9b2dc\n", + " health: HEALTH_WARN\n", + " 2 MDSs report slow metadata IOs\n", + " Reduced data availability: 48 pgs inactive\n", + " Degraded data redundancy: 66/196 objects degraded (33.673%), 11 pgs degraded, 48 pgs undersized\n", + " \n", + " services:\n", + " mon: 2 daemons, quorum a,b,c (age 4w)\n", + " mgr: b(active, since 9d), standbys: c, a\n", + " mds: 1/1 daemons up, 1 hot standby\n", + " osd: 3 osds: 3 up (since 3w), 3 in (since 3w)\n", + " \n", + " data:\n", + " volumes: 1/1 healthy\n", + " pools: 4 pools, 112 pgs\n", + " objects: 27 objects, 9.1 MiB\n", + " usage: 425 MiB used, 2.2 TiB / 2.2 TiB avail\n", + " pgs: 112 active+clean\n", + " \n", + " io:\n", + " client: 1.2 KiB/s rd, 2 op/s rd, 0 op/s wr\n", + " \n" +] + +# fmt: off +ceph_s_health_ok = [ + " cluster:\n", + " id: 8abb43ce-6775-4a1a-99c4-12f37101410e\n", + " health: HEALTH_OK\n", + " \n", + " services:\n", + " mon: 3 daemons, quorum a,b,c (age 4w)\n", + " mgr: b(active, since 9d), standbys: c, a\n", + " mds: 1/1 daemons up, 1 hot standby\n", + " osd: 5 osds: 5 up (since 3w), 5 in (since 3w)\n", + " \n", + " data:\n", + " volumes: 1/1 healthy\n", + " pools: 4 pools, 112 pgs\n", + " objects: 27 objects, 9.1 MiB\n", + " usage: 425 MiB used, 2.2 TiB / 2.2 TiB avail\n", + " pgs: 112 active+clean\n", + " \n", + " io:\n", + " client: 1.2 KiB/s rd, 2 op/s rd, 0 op/s wr\n", + " \n" +] + + +def test_ceph_status_output(): + """ + Tests ceph_status_output functions + + include: + is_ceph_healthy() + get_ceph_osd_count() + get_ceph_mon_count() + + """ + ceph_s_health_ok_output = CephStatusOutput(ceph_s_health_ok) + cluster_object = ceph_s_health_ok_output.get_ceph_cluster_output().get_ceph_cluster_object() + assert cluster_object.get_id() == "8abb43ce-6775-4a1a-99c4-12f37101410e" + assert cluster_object.get_health() == "HEALTH_OK" + assert ceph_s_health_ok_output.get_ceph_osd_count() == 5 + assert ceph_s_health_ok_output.get_ceph_mon_count() == 3 + + ceph_s_health_status = ceph_s_health_ok_output.is_ceph_healthy() + if not ceph_s_health_status: + raise ValueError("Error output") + + ceph_s_health_warn_output = CephStatusOutput(ceph_s_health_warning) + cluster_object = ceph_s_health_warn_output.get_ceph_cluster_output().get_ceph_cluster_object() + assert cluster_object.get_id() == "001ab3d8-1f02-4294-b594-e2b216b9b2dc" + assert cluster_object.get_health() == "HEALTH_WARN; 2 MDSs report slow metadata IOs; Reduced data availability: 48 pgs inactive; Degraded data redundancy: 66/196 objects degraded (33.673%), 11 pgs degraded, 48 pgs undersized" + assert ceph_s_health_warn_output.get_ceph_osd_count() == 3 + assert ceph_s_health_warn_output.get_ceph_mon_count() == 2 + ceph_s_health_status = ceph_s_health_warn_output.is_ceph_healthy() + if ceph_s_health_status: + raise ValueError("Error output")