From 195f3ed93b5171b3559e196a087214aa85d1d794 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 12 Apr 2024 13:26:45 -0300 Subject: [PATCH] Implement key rotation for OSD's This patchset implements key rotation for OSD units. The monitor on which this action is called will set the 'pending_key' field in the relation databag, which specifies the OSD id and new key. On their side, OSD units will check this field and compare against the OSD ids that they maintain to tell whether they need to rotate the key or not. Change-Id: Ief5afdea2b8449adbe14c7e838330e2f2be1cfd2 --- src/ceph_hooks.py | 1 + src/ops_actions/rotate_key.py | 111 +++++++++++++++++++++++++++++++- unit_tests/test_ceph_actions.py | 49 +++++++++++++- 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/src/ceph_hooks.py b/src/ceph_hooks.py index 101b1066..6eab0936 100755 --- a/src/ceph_hooks.py +++ b/src/ceph_hooks.py @@ -871,6 +871,7 @@ def osd_relation(relid=None, unit=None, reprocess_broker_requests=False): log('mon cluster in quorum - providing fsid & keys') public_addr = get_public_addr() data = { + 'pending_key': '', 'fsid': leader_get('fsid'), 'osd_bootstrap_key': ceph.get_osd_bootstrap_key(), 'auth': 'cephx', diff --git a/src/ops_actions/rotate_key.py b/src/ops_actions/rotate_key.py index 8e473bba..1e4ba829 100644 --- a/src/ops_actions/rotate_key.py +++ b/src/ops_actions/rotate_key.py @@ -126,6 +126,111 @@ def _handle_mds_key_rotation(entity, event, model): event.set_results({'message': 'success'}) +def _get_osd_tree(): + out = subprocess.check_output(["sudo", "ceph", "osd", "dump", + "--format=json"]) + return json.loads(out.decode("utf8")).get("osds", ()) + + +def _clean_address(addr): + ix = addr.find(":") + return addr if ix < 0 else addr[0:ix] + + +def _get_osd_addrs(osd_id, tree=None): + if tree is None: + tree = _get_osd_tree() + + for osd in tree: + if osd.get("osd") != osd_id: + continue + + return [_clean_address(osd[x]) + for x in ("public_addr", "cluster_addr") + if x in osd] + + +def _get_unit_addr(unit, rel_id): + out = subprocess.check_output(["relation-get", "--format=json", + "-r", str(rel_id), "private-address", unit]) + return out.decode("utf8").replace('"', '').strip() + + +def _find_osd_unit(relations, model, osd_id, tree): + addrs = _get_osd_addrs(osd_id, tree) + if not addrs: + return None + + for relation in relations: + for unit in relation.units: + if _get_unit_addr(unit.name, relation.id) in addrs: + return relation.data[model.unit] + + +def _handle_osd_key_rotation(entity, event, model, tree=None): + osd_rels = model.relations.get("osd") + if not osd_rels: + event.fail("No OSD relations found") + return + + if tree is None: + tree = _get_osd_tree() + + osd_id = int(entity[4:]) + bag = _find_osd_unit(osd_rels, model, osd_id, tree) + if bag is not None: + key = _create_key(entity, event) + bag["pending_key"] = json.dumps({osd_id: key}) + event.set_results({"message": "success"}) + else: + event.fail("No OSD matching entity %s found" % entity) + + +def _add_osd_rotation(rotations, new_bag, osd_id, new_key): + # NOTE(lmlg): We can't use sets or dicts for relation databags, as they + # are mutable and don't implement a __hash__ method. So we use a simple + # (bag, dict) array to map the rotations. + elem = {osd_id: new_key} + for bag, data in rotations: + if bag is new_bag: + data.update(elem) + return + + rotations.append((new_bag, elem)) + + +def _get_osd_ids(): + ret = subprocess.check_output(["sudo", "ceph", "osd", "ls"]) + return ret.decode("utf8").split("\n") + + +def _rotate_all_osds(event, model): + tree = _get_osd_tree() + osd_rels = model.relations.get("osd") + ret = [] + + if not osd_rels: + event.fail("No OSD relations found") + return + + for osd_id in _get_osd_ids(): + osd_id = osd_id.strip() + if not osd_id: + continue + + bag = _find_osd_unit(osd_rels, model, int(osd_id), tree) + if bag is None: + continue + + key = _create_key("osd." + osd_id, event) + _add_osd_rotation(ret, bag, osd_id, key) + + for bag, elem in ret: + bag["pending_key"] = json.dumps(elem) + + event.set_results({"message": "success"}) + + def rotate_key(event, model=None) -> None: """Rotate the key of the specified entity.""" entity = event.params.get("entity") @@ -150,9 +255,13 @@ def rotate_key(event, model=None) -> None: _replace_keyring_file(path, entity, key, event) _restart_daemon("ceph-mgr@%s.service" % entity[4:], event) event.set_results({"message": "success"}) - elif entity.startswith('client.rgw.'): + elif entity.startswith("client.rgw."): _handle_rgw_key_rotation(entity, event, model) elif entity.startswith('mds.'): _handle_mds_key_rotation(entity, event, model) + elif entity == "osd": + _rotate_all_osds(event, model) + elif entity.startswith("osd."): + _handle_osd_key_rotation(entity, event, model) else: event.fail("Unknown entity: %s" % entity) diff --git a/unit_tests/test_ceph_actions.py b/unit_tests/test_ceph_actions.py index b3be8164..6a4b77db 100644 --- a/unit_tests/test_ceph_actions.py +++ b/unit_tests/test_ceph_actions.py @@ -316,6 +316,21 @@ MGR_KEYRING_FILE = """ key = old-key """ +OSD_DUMP = b""" +{ + "osds": [ + { + "osd": 0, + "public_addr": "10.5.2.40:6801/13869" + }, + { + "osd": 1, + "public_addr": "10.5.0.160:6801/9017" + } + ] +} +""" + class RotateKey(test_utils.CharmTestCase): """Run tests for action.""" @@ -346,7 +361,7 @@ class RotateKey(test_utils.CharmTestCase): check_output.return_value = b'[{"pending_key": "new-key"}]' event = test_utils.MockActionEvent({'entity': 'mgr.host-1'}) - self.harness.charm.on_rotate_key_action(event) + rotate_key.rotate_key(event) event.set_results.assert_called_with({'message': 'success'}) listdir.assert_called_once_with('/var/lib/ceph/mgr') @@ -356,3 +371,35 @@ class RotateKey(test_utils.CharmTestCase): calls = any(x for x in _open.mock_calls if any(p is not None and 'new-key' in p for p in x.args)) self.assertTrue(calls) + + @mock.patch.object(rotate_key, '_create_key') + @mock.patch.object(rotate_key.subprocess, 'check_output') + def test_rotate_osd_key(self, check_output, create_key): + def _check_output_inner(args): + if args == ['sudo', 'ceph', 'osd', 'dump', '--format=json']: + return OSD_DUMP + elif args[5] == 'ceph-osd/0': + return b'10.5.2.40' + else: + return b'10.5.0.160' + + check_output.side_effect = _check_output_inner + create_key.return_value = 'some-key' + + unit0 = mock.MagicMock() + unit0.name = 'ceph-osd/0' + unit1 = mock.MagicMock() + unit1.name = 'ceph-osd/1' + + relations = mock.MagicMock() + relations.units = [unit0, unit1] + relations.data = {'ceph-mon/0': {}} + + model = mock.MagicMock() + model.relations = {'osd': [relations]} + model.unit = 'ceph-mon/0' + + event = test_utils.MockActionEvent({'entity': 'osd.1'}) + rotate_key.rotate_key(event, model) + self.assertEqual(relations.data['ceph-mon/0'], + {'pending_key': '{"1": "some-key"}'})