Implement key rotation for OSD's

This patchset implements key rotation for OSD units. The monitor
on which this action is called will set the 'pending_key' field
in the relation databag, which specifies the OSD id and new key.
On their side, OSD units will check this field and compare against
the OSD ids that they maintain to tell whether they need to
rotate the key or not.

Change-Id: Ief5afdea2b8449adbe14c7e838330e2f2be1cfd2

src/ceph_hooks.py

@@ -871,6 +871,7 @@ def osd_relation(relid=None, unit=None, reprocess_broker_requests=False):
         log('mon cluster in quorum - providing fsid & keys')
         public_addr = get_public_addr()
         data = {
+            'pending_key': '',
             'fsid': leader_get('fsid'),
             'osd_bootstrap_key': ceph.get_osd_bootstrap_key(),
             'auth': 'cephx',

src/ops_actions/rotate_key.py

@@ -126,6 +126,111 @@ def _handle_mds_key_rotation(entity, event, model):
     event.set_results({'message': 'success'})
 
 
+def _get_osd_tree():
+    out = subprocess.check_output(["sudo", "ceph", "osd", "dump",
+                                   "--format=json"])
+    return json.loads(out.decode("utf8")).get("osds", ())
+
+
+def _clean_address(addr):
+    ix = addr.find(":")
+    return addr if ix < 0 else addr[0:ix]
+
+
+def _get_osd_addrs(osd_id, tree=None):
+    if tree is None:
+        tree = _get_osd_tree()
+
+    for osd in tree:
+        if osd.get("osd") != osd_id:
+            continue
+
+        return [_clean_address(osd[x])
+                for x in ("public_addr", "cluster_addr")
+                if x in osd]
+
+
+def _get_unit_addr(unit, rel_id):
+    out = subprocess.check_output(["relation-get", "--format=json",
+                                   "-r", str(rel_id), "private-address", unit])
+    return out.decode("utf8").replace('"', '').strip()
+
+
+def _find_osd_unit(relations, model, osd_id, tree):
+    addrs = _get_osd_addrs(osd_id, tree)
+    if not addrs:
+        return None
+
+    for relation in relations:
+        for unit in relation.units:
+            if _get_unit_addr(unit.name, relation.id) in addrs:
+                return relation.data[model.unit]
+
+
+def _handle_osd_key_rotation(entity, event, model, tree=None):
+    osd_rels = model.relations.get("osd")
+    if not osd_rels:
+        event.fail("No OSD relations found")
+        return
+
+    if tree is None:
+        tree = _get_osd_tree()
+
+    osd_id = int(entity[4:])
+    bag = _find_osd_unit(osd_rels, model, osd_id, tree)
+    if bag is not None:
+        key = _create_key(entity, event)
+        bag["pending_key"] = json.dumps({osd_id: key})
+        event.set_results({"message": "success"})
+    else:
+        event.fail("No OSD matching entity %s found" % entity)
+
+
+def _add_osd_rotation(rotations, new_bag, osd_id, new_key):
+    # NOTE(lmlg): We can't use sets or dicts for relation databags, as they
+    # are mutable and don't implement a __hash__ method. So we use a simple
+    # (bag, dict) array to map the rotations.
+    elem = {osd_id: new_key}
+    for bag, data in rotations:
+        if bag is new_bag:
+            data.update(elem)
+            return
+
+    rotations.append((new_bag, elem))
+
+
+def _get_osd_ids():
+    ret = subprocess.check_output(["sudo", "ceph", "osd", "ls"])
+    return ret.decode("utf8").split("\n")
+
+
+def _rotate_all_osds(event, model):
+    tree = _get_osd_tree()
+    osd_rels = model.relations.get("osd")
+    ret = []
+
+    if not osd_rels:
+        event.fail("No OSD relations found")
+        return
+
+    for osd_id in _get_osd_ids():
+        osd_id = osd_id.strip()
+        if not osd_id:
+            continue
+
+        bag = _find_osd_unit(osd_rels, model, int(osd_id), tree)
+        if bag is None:
+            continue
+
+        key = _create_key("osd." + osd_id, event)
+        _add_osd_rotation(ret, bag, osd_id, key)
+
+    for bag, elem in ret:
+        bag["pending_key"] = json.dumps(elem)
+
+    event.set_results({"message": "success"})
+
+
 def rotate_key(event, model=None) -> None:
     """Rotate the key of the specified entity."""
     entity = event.params.get("entity")
@@ -150,9 +255,13 @@ def rotate_key(event, model=None) -> None:
         _replace_keyring_file(path, entity, key, event)
         _restart_daemon("ceph-mgr@%s.service" % entity[4:], event)
         event.set_results({"message": "success"})
-    elif entity.startswith('client.rgw.'):
+    elif entity.startswith("client.rgw."):
         _handle_rgw_key_rotation(entity, event, model)
     elif entity.startswith('mds.'):
         _handle_mds_key_rotation(entity, event, model)
+    elif entity == "osd":
+        _rotate_all_osds(event, model)
+    elif entity.startswith("osd."):
+        _handle_osd_key_rotation(entity, event, model)
     else:
         event.fail("Unknown entity: %s" % entity)

unit_tests/test_ceph_actions.py

@@ -316,6 +316,21 @@ MGR_KEYRING_FILE = """
   key = old-key
 """
 
+OSD_DUMP = b"""
+{
+    "osds": [
+        {
+            "osd": 0,
+            "public_addr": "10.5.2.40:6801/13869"
+        },
+        {
+            "osd": 1,
+            "public_addr": "10.5.0.160:6801/9017"
+        }
+    ]
+}
+"""
+
 
 class RotateKey(test_utils.CharmTestCase):
     """Run tests for action."""
@@ -346,7 +361,7 @@ class RotateKey(test_utils.CharmTestCase):
         check_output.return_value = b'[{"pending_key": "new-key"}]'
 
         event = test_utils.MockActionEvent({'entity': 'mgr.host-1'})
-        self.harness.charm.on_rotate_key_action(event)
+        rotate_key.rotate_key(event)
 
         event.set_results.assert_called_with({'message': 'success'})
         listdir.assert_called_once_with('/var/lib/ceph/mgr')
@@ -356,3 +371,35 @@ class RotateKey(test_utils.CharmTestCase):
         calls = any(x for x in _open.mock_calls
                     if any(p is not None and 'new-key' in p for p in x.args))
         self.assertTrue(calls)
+
+    @mock.patch.object(rotate_key, '_create_key')
+    @mock.patch.object(rotate_key.subprocess, 'check_output')
+    def test_rotate_osd_key(self, check_output, create_key):
+        def _check_output_inner(args):
+            if args == ['sudo', 'ceph', 'osd', 'dump', '--format=json']:
+                return OSD_DUMP
+            elif args[5] == 'ceph-osd/0':
+                return b'10.5.2.40'
+            else:
+                return b'10.5.0.160'
+
+        check_output.side_effect = _check_output_inner
+        create_key.return_value = 'some-key'
+
+        unit0 = mock.MagicMock()
+        unit0.name = 'ceph-osd/0'
+        unit1 = mock.MagicMock()
+        unit1.name = 'ceph-osd/1'
+
+        relations = mock.MagicMock()
+        relations.units = [unit0, unit1]
+        relations.data = {'ceph-mon/0': {}}
+
+        model = mock.MagicMock()
+        model.relations = {'osd': [relations]}
+        model.unit = 'ceph-mon/0'
+
+        event = test_utils.MockActionEvent({'entity': 'osd.1'})
+        rotate_key.rotate_key(event, model)
+        self.assertEqual(relations.data['ceph-mon/0'],
+                         {'pending_key': '{"1": "some-key"}'})