Add support for object gateway integration

To add support for object gateways this change adds the
radosgw-user interface.

The focal bundle has one rados gateway application as more
than that is not supported on Octopus. The hersute bundle
has two rados gateway applications but does not contain
the lma applications as telegraf is not supported on
hersute.

Depends-On: Ieff1943b02f490559ccd245f60b744fb76a5d832
Change-Id: I38939b9938a5ba2ed6e3fb489f66a255f7aa8fe4

README.md

@@ -69,6 +69,14 @@ To enable Prometheus alerting, add the following relations:
     juju relate ceph-dashboard:alertmanager-service prometheus-alertmanager:alertmanager-service
     juju relate prometheus:alertmanager-service prometheus-alertmanager:alertmanager-service
 
+## Object Gateway
+
+To enable object gateway management add the following relation:
+
+    juju relate ceph-dashboard:radosgw-dashboard ceph-radosgw:radosgw-user
+
+NOTE: On Octopus or earlier the dashboard can only be related to one ceph-radosgw application.
+
 <!-- LINKS -->
 
 [ceph-dashboard]: https://docs.ceph.com/en/latest/mgr/dashboard/

metadata.yaml

@@ -16,6 +16,7 @@ subordinate: true
 series:
 - focal
 - groovy
+- hirsute
 requires:
   dashboard:
     interface: ceph-dashboard
@@ -28,6 +29,8 @@ requires:
     interface: http
   prometheus:
     interface: http
+  radosgw-dashboard:
+    interface: radosgw-user
 provides:
   grafana-dashboard:
     interface: grafana-dashboard

osci.yaml

@@ -3,8 +3,27 @@
       - charm-unit-jobs
     check:
       jobs:
-      - focal
+      - focal-octopus
+      - hirsute-pacific
     vars:
       needs_charm_build: true
       charm_build_name: ceph-dashboard
       build_type: charmcraft
+- job:
+    name: focal-octopus
+    parent: func-target
+    dependencies:
+      - osci-lint
+      - tox-py35
+      - tox-py36
+      - tox-py37
+      - tox-py38
+    vars:
+      tox_extra_args: focal
+- job:
+    name: hirsute-pacific
+    parent: func-target
+    dependencies: &smoke-jobs
+      - focal-octopus
+    vars:
+      tox_extra_args: hirsute

src/charm.py

@@ -29,6 +29,7 @@ import interface_dashboard
 import interface_api_endpoints
 import interface_grafana_dashboard
 import interface_http
+import interface_radosgw_user
 import cryptography.hazmat.primitives.serialization as serialization
 import charms_ceph.utils as ceph_utils
 import charmhelpers.core.host as ch_host
@@ -161,6 +162,10 @@ class CephDashboardCharm(ops_openstack.core.OSBaseCharm):
         self.ca_client = ca_client.CAClient(
             self,
             'certificates')
+        self.radosgw_user = interface_radosgw_user.RadosGWUserRequires(
+            self,
+            'radosgw-dashboard',
+            request_system_role=True)
         self.framework.observe(
             self.mon.on.mon_ready,
             self._configure_dashboard)
@@ -170,6 +175,9 @@ class CephDashboardCharm(ops_openstack.core.OSBaseCharm):
         self.framework.observe(
             self.ca_client.on.tls_server_config_ready,
             self._configure_dashboard)
+        self.framework.observe(
+            self.radosgw_user.on.gw_user_ready,
+            self._configure_dashboard)
         self.framework.observe(self.on.add_user_action, self._add_user_action)
         self.ingress = interface_api_endpoints.APIEndpointsRequires(
             self,
@@ -211,6 +219,50 @@ class CephDashboardCharm(ops_openstack.core.OSBaseCharm):
             logging.info(
                 "register_grafana_dashboard: {}".format(dash_file))
 
+    def _update_legacy_radosgw_creds(self, access_key: str,
+                                     secret_key: str) -> None:
+        """Update dashboard db with access & secret key for rados gateways.
+
+        This method uses the legacy format which only supports one gateway.
+        """
+        self._apply_file_setting('set-rgw-api-access-key', access_key)
+        self._apply_file_setting('set-rgw-api-secret-key', secret_key)
+
+    def _update_multi_radosgw_creds(self, creds: str) -> None:
+        """Update dashboard db with access & secret key for rados gateway."""
+        access_keys = {c['daemon_id']: c['access_key'] for c in creds}
+        secret_keys = {c['daemon_id']: c['secret_key'] for c in creds}
+        self._apply_file_setting(
+            'set-rgw-api-access-key',
+            json.dumps(access_keys))
+        self._apply_file_setting(
+            'set-rgw-api-secret-key',
+            json.dumps(secret_keys))
+
+    def _support_multiple_gateways(self) -> bool:
+        """Check if version of dashboard supports multiple rados gateways"""
+        return ch_host.cmp_pkgrevno('ceph-common', '16.0') > 0
+
+    def _manage_radosgw(self) -> None:
+        """Register rados gateways in dashboard db"""
+        if self.unit.is_leader():
+            creds = self.radosgw_user.get_user_creds()
+            if len(creds) < 1:
+                logging.info("No object gateway creds found")
+                return
+            if self._support_multiple_gateways():
+                self._update_multi_radosgw_creds(creds)
+            else:
+                if len(creds) > 1:
+                    logging.error(
+                        "Cannot enable object gateway support. Ceph release "
+                        "does not support multiple object gateways in the "
+                        "dashboard")
+                else:
+                    self._update_legacy_radosgw_creds(
+                        creds[0]['access_key'],
+                        creds[0]['secret_key'])
+
     def _on_ca_available(self, _) -> None:
         """Request TLS certificates."""
         addresses = set()
@@ -280,16 +332,31 @@ class CephDashboardCharm(ops_openstack.core.OSBaseCharm):
         ceph_utils.mgr_disable_dashboard()
         ceph_utils.mgr_enable_dashboard()
 
-    def _run_cmd(self, cmd: List[str]) -> None:
+    def _run_cmd(self, cmd: List[str]) -> str:
         """Run command in subprocess
 
         `cmd` The command to run
         """
         try:
-            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            return output.decode('UTF-8')
         except subprocess.CalledProcessError as exc:
             logging.exception("Command failed: {}".format(exc.output))
 
+    def _apply_setting(self, ceph_setting: str, value: List[str]) -> str:
+        """Apply a dashboard setting"""
+        cmd = ['ceph', 'dashboard', ceph_setting]
+        cmd.extend(value)
+        return self._run_cmd(cmd)
+
+    def _apply_file_setting(self, ceph_setting: str,
+                            file_contents: str) -> str:
+        """Apply a setting via a file"""
+        with tempfile.NamedTemporaryFile(mode='w', delete=True) as _file:
+            _file.write(file_contents)
+            _file.flush()
+            return self._apply_setting(ceph_setting, ['-i', _file.name])
+
     def _apply_ceph_config_from_charm_config(self) -> None:
         """Read charm config and apply settings to dashboard config"""
         for option in self.CHARM_TO_CEPH_OPTIONS:
@@ -342,6 +409,7 @@ class CephDashboardCharm(ops_openstack.core.OSBaseCharm):
                     'ceph', 'dashboard', 'set-prometheus-api-host',
                     prometheus_ep])
         self._register_dashboards()
+        self._manage_radosgw()
         self._stored.is_started = True
         self.update_status()
 

src/interface_radosgw_user.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from ops.framework import (
+    StoredState,
+    EventBase,
+    ObjectEvents,
+    EventSource,
+    Object)
+
+
+class RadosGWUserEvent(EventBase):
+    pass
+
+
+class RadosGWUserEvents(ObjectEvents):
+    gw_user_ready = EventSource(RadosGWUserEvent)
+
+
+class RadosGWUserRequires(Object):
+
+    on = RadosGWUserEvents()
+    _stored = StoredState()
+
+    def __init__(self, charm, relation_name, request_system_role=False):
+        super().__init__(charm, relation_name)
+        self.relation_name = relation_name
+        self.request_system_role = request_system_role
+        self.framework.observe(
+            charm.on[self.relation_name].relation_joined,
+            self.request_user)
+        self.framework.observe(
+            charm.on[self.relation_name].relation_changed,
+            self._on_relation_changed)
+
+    def request_user(self, event):
+        if self.model.unit.is_leader():
+            for relation in self.framework.model.relations[self.relation_name]:
+                relation.data[self.model.app]['system-role'] = json.dumps(
+                    self.request_system_role)
+
+    def get_user_creds(self):
+        creds = []
+        for relation in self.framework.model.relations[self.relation_name]:
+            app_data = relation.data[relation.app]
+            for unit in relation.units:
+                unit_data = relation.data[unit]
+                cred_data = {
+                    'access_key': app_data.get('access-key'),
+                    'secret_key': app_data.get('secret-key'),
+                    'uid': app_data.get('uid'),
+                    'daemon_id': unit_data.get('daemon-id')}
+                if all(cred_data.values()):
+                    creds.append(cred_data)
+        creds = sorted(creds, key=lambda k: k['daemon_id'])
+        return creds
+
+    def _on_relation_changed(self, event):
+        """Handle the relation-changed event."""
+        if self.get_user_creds():
+            self.on.gw_user_ready.emit()

tests/bundles/focal.yaml

@@ -47,6 +47,9 @@ applications:
   prometheus-alertmanager:
     charm: cs:prometheus-alertmanager
     num_units: 1
+  ceph-radosgw:
+    charm: cs:~openstack-charmers-next/ceph-radosgw
+    num_units: 3
 relations:
   - - 'ceph-osd:mon'
     - 'ceph-mon:osd'
@@ -80,3 +83,9 @@ relations:
     - 'prometheus:website'
   - - 'prometheus:alertmanager-service'
     - 'prometheus-alertmanager:alertmanager-service'
+  - - 'ceph-radosgw:mon'
+    - 'ceph-mon:radosgw'
+  - - 'ceph-radosgw:certificates'
+    - 'vault:certificates'
+  - - 'ceph-dashboard:radosgw-dashboard'
+    - 'ceph-radosgw:radosgw-user'

tests/bundles/hirsute.yaml

@@ -0,0 +1,63 @@
+local_overlay_enabled: False
+series: hirsute
+applications:
+  ceph-osd:
+    charm: cs:~openstack-charmers-next/ceph-osd
+    num_units: 6
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      osd-devices: '/dev/test-non-existent'
+  ceph-mon:
+    charm: cs:~openstack-charmers-next/ceph-mon
+    num_units: 3
+    options:
+      monitor-count: '3'
+  vault:
+    num_units: 1
+    charm: cs:~openstack-charmers-next/vault
+  mysql-innodb-cluster:
+    charm: cs:~openstack-charmers-next/mysql-innodb-cluster
+    constraints: mem=3072M
+    num_units: 3
+  vault-mysql-router:
+    charm: cs:~openstack-charmers-next/mysql-router
+  ceph-dashboard:
+    charm: ../../ceph-dashboard.charm
+    options:
+      public-hostname: 'ceph-dashboard.zaza.local'
+  ceph-radosgw-east:
+    charm: cs:~openstack-charmers-next/ceph-radosgw
+    num_units: 3
+    options:
+      pool-prefix: east
+      region: east
+  ceph-radosgw-west:
+    charm: cs:~openstack-charmers-next/ceph-radosgw
+    num_units: 3
+    options:
+      pool-prefix: west
+      region: west
+relations:
+  - - 'ceph-osd:mon'
+    - 'ceph-mon:osd'
+  - - 'vault:shared-db'
+    - 'vault-mysql-router:shared-db'
+  - - 'vault-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+  - - 'ceph-dashboard:dashboard'
+    - 'ceph-mon:dashboard'
+  - - 'ceph-dashboard:certificates'
+    - 'vault:certificates'
+  - - 'ceph-radosgw-east:mon'
+    - 'ceph-mon:radosgw'
+  - - 'ceph-radosgw-east:certificates'
+    - 'vault:certificates'
+  - - 'ceph-dashboard:radosgw-dashboard'
+    - 'ceph-radosgw-east:radosgw-user'
+  - - 'ceph-radosgw-west:mon'
+    - 'ceph-mon:radosgw'
+  - - 'ceph-radosgw-west:certificates'
+    - 'vault:certificates'
+  - - 'ceph-dashboard:radosgw-dashboard'
+    - 'ceph-radosgw-west:radosgw-user'

tests/tests.yaml

@@ -1,6 +1,7 @@
 charm_name: ceph-dasboard
 gate_bundles:
   - focal
+  - hirsute
 smoke_bundles:
   - focal
 configure:
@@ -12,7 +13,7 @@ tests:
 target_deploy_status:
   ceph-dashboard:
     workload-status: blocked
-    workload-status-message-regex: "No certificates found|Charm config option"
+    workload-status-message-regex: "No certificates found|Charm config option|Unit is ready"
   vault:
     workload-status: blocked
     workload-status-message-prefix: Vault needs to be initialized
@@ -25,3 +26,6 @@ target_deploy_status:
   telegraf:
     workload-status: active
     workload-status-message-prefix: Monitoring
+tests_options:
+  force_deploy:
+    - hirsute

unit_tests/test_ceph_dashboard_charm.py

@@ -21,7 +21,7 @@ import sys
 sys.path.append('lib')  # noqa
 sys.path.append('src')  # noqa
 
-from mock import call, patch, MagicMock
+from mock import ANY, call, patch, MagicMock
 
 from ops.testing import Harness, _TestingModelBackend
 from ops.model import (
@@ -155,6 +155,7 @@ class TestCephDashboardCharmBase(CharmTestCase):
 
     PATCHES = [
         'ceph_utils',
+        'ch_host',
         'socket',
         'subprocess',
         'ch_host',
@@ -464,6 +465,148 @@ class TestCephDashboardCharmBase(CharmTestCase):
         self.ceph_utils.mgr_disable_dashboard.assert_called_once_with()
         self.ceph_utils.mgr_enable_dashboard.assert_called_once_with()
 
+    def test_rados_gateway(self):
+        self.ceph_utils.is_dashboard_enabled.return_value = True
+        self.ch_host.cmp_pkgrevno.return_value = 1
+        mon_rel_id = self.harness.add_relation('dashboard', 'ceph-mon')
+        rel_id = self.harness.add_relation('radosgw-dashboard', 'ceph-radosgw')
+        self.harness.begin()
+        self.harness.set_leader()
+        self.harness.add_relation_unit(
+            mon_rel_id,
+            'ceph-mon/0')
+        self.harness.update_relation_data(
+            mon_rel_id,
+            'ceph-mon/0',
+            {
+                'mon-ready': 'True'})
+        self.harness.add_relation_unit(
+            rel_id,
+            'ceph-radosgw/0')
+        self.harness.add_relation_unit(
+            rel_id,
+            'ceph-radosgw/1')
+        self.harness.update_relation_data(
+            rel_id,
+            'ceph-radosgw/0',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-3'})
+        self.harness.update_relation_data(
+            rel_id,
+            'ceph-radosgw/1',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-4'})
+        self.harness.update_relation_data(
+            rel_id,
+            'ceph-radosgw',
+            {
+                'access-key': 'XNUZVPL364U0BL1OXWJZ',
+                'secret-key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                'uid': 'radosgw-user-9'})
+        self.subprocess.check_output.assert_has_calls([
+            call(['ceph', 'dashboard', 'set-rgw-api-access-key', '-i', ANY],
+                 stderr=self.subprocess.STDOUT),
+            call().decode('UTF-8'),
+            call(['ceph', 'dashboard', 'set-rgw-api-secret-key', '-i', ANY],
+                 stderr=self.subprocess.STDOUT),
+            call().decode('UTF-8'),
+        ])
+
+    def test_rados_gateway_multi_relations_pacific(self):
+        self.ceph_utils.is_dashboard_enabled.return_value = True
+        self.ch_host.cmp_pkgrevno.return_value = 1
+        rel_id1 = self.harness.add_relation('radosgw-dashboard', 'ceph-eu')
+        rel_id2 = self.harness.add_relation('radosgw-dashboard', 'ceph-us')
+        mon_rel_id = self.harness.add_relation('dashboard', 'ceph-mon')
+        self.harness.begin()
+        self.harness.set_leader()
+        self.harness.add_relation_unit(
+            mon_rel_id,
+            'ceph-mon/0')
+        self.harness.update_relation_data(
+            mon_rel_id,
+            'ceph-mon/0',
+            {
+                'mon-ready': 'True'})
+        self.harness.add_relation_unit(
+            rel_id1,
+            'ceph-eu/0')
+        self.harness.add_relation_unit(
+            rel_id2,
+            'ceph-us/0')
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu/0',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-3'})
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us/0',
+            {
+                'daemon-id': 'juju-dddddd-zaza-sdfsfsfs-4'})
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu',
+            {
+                'access-key': 'XNUZVPL364U0BL1OXWJZ',
+                'secret-key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                'uid': 'radosgw-user-9'})
+        self.subprocess.check_output.reset_mock()
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us',
+            {
+                'access-key': 'JGHKJGDKJGJGJHGYYYYM',
+                'secret-key': 'iljkdfhHKHKd88LKxNLSKDiijfjfjfldjfjlf44',
+                'uid': 'radosgw-user-10'})
+        self.subprocess.check_output.assert_has_calls([
+            call(['ceph', 'dashboard', 'set-rgw-api-access-key', '-i', ANY],
+                 stderr=self.subprocess.STDOUT),
+            call().decode('UTF-8'),
+            call(['ceph', 'dashboard', 'set-rgw-api-secret-key', '-i', ANY],
+                 stderr=self.subprocess.STDOUT),
+            call().decode('UTF-8'),
+        ])
+
+    def test_rados_gateway_multi_relations_octopus(self):
+        self.ch_host.cmp_pkgrevno.return_value = -1
+        rel_id1 = self.harness.add_relation('radosgw-dashboard', 'ceph-eu')
+        rel_id2 = self.harness.add_relation('radosgw-dashboard', 'ceph-us')
+        self.harness.begin()
+        self.harness.set_leader()
+        self.harness.add_relation_unit(
+            rel_id1,
+            'ceph-eu/0')
+        self.harness.add_relation_unit(
+            rel_id2,
+            'ceph-us/0')
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu/0',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-3'})
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us/0',
+            {
+                'daemon-id': 'juju-dddddd-zaza-sdfsfsfs-4'})
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu',
+            {
+                'access-key': 'XNUZVPL364U0BL1OXWJZ',
+                'secret-key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                'uid': 'radosgw-user-9'})
+        self.subprocess.check_output.reset_mock()
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us',
+            {
+                'access-key': 'JGHKJGDKJGJGJHGYYYYM',
+                'secret-key': 'iljkdfhHKHKd88LKxNLSKDiijfjfjfldjfjlf44',
+                'uid': 'radosgw-user-10'})
+        self.assertFalse(self.subprocess.check_output.called)
+
     @patch.object(charm.secrets, 'choice')
     def test__gen_user_password(self, _choice):
         self.harness.begin()

unit_tests/test_interface_radosgw_user.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import sys
+sys.path.append('lib')  # noqa
+sys.path.append('src')  # noqa
+from ops.testing import Harness
+from ops.charm import CharmBase
+import interface_radosgw_user
+
+
+class TestRadosGWUserRequires(unittest.TestCase):
+
+    class MyCharm(CharmBase):
+
+        def __init__(self, *args):
+            super().__init__(*args)
+            self.seen_events = []
+            self.radosgw_user = interface_radosgw_user.RadosGWUserRequires(
+                self,
+                'radosgw-dashboard')
+
+            self.framework.observe(
+                self.radosgw_user.on.gw_user_ready,
+                self._log_event)
+
+        def _log_event(self, event):
+            self.seen_events.append(type(event).__name__)
+
+    def setUp(self):
+        super().setUp()
+        self.harness = Harness(
+            self.MyCharm,
+            meta='''
+name: my-charm
+requires:
+  radosgw-dashboard:
+    interface: radosgw-user
+'''
+        )
+
+    def test_init(self):
+        self.harness.begin()
+        self.assertEqual(
+            self.harness.charm.radosgw_user.relation_name,
+            'radosgw-dashboard')
+
+    def test_add_radosgw_dashboard_relation(self):
+        rel_id1 = self.harness.add_relation('radosgw-dashboard', 'ceph-eu')
+        rel_id2 = self.harness.add_relation('radosgw-dashboard', 'ceph-us')
+        self.harness.begin()
+        self.assertEqual(
+            self.harness.charm.seen_events,
+            [])
+        self.harness.set_leader()
+        self.harness.add_relation_unit(
+            rel_id1,
+            'ceph-eu/0')
+        self.harness.add_relation_unit(
+            rel_id1,
+            'ceph-eu/1')
+        self.harness.add_relation_unit(
+            rel_id2,
+            'ceph-us/0')
+        self.harness.add_relation_unit(
+            rel_id2,
+            'ceph-us/1')
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu/0',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-3'})
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu/1',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-4'})
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us/0',
+            {
+                'daemon-id': 'juju-dddddd-zaza-sdfsfsfs-4'})
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us/1',
+            {
+                'daemon-id': 'juju-dddddd-zaza-sdfsfsfs-5'})
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu',
+            {
+                'access-key': 'XNUZVPL364U0BL1OXWJZ',
+                'secret-key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                'uid': 'radosgw-user-9'})
+        self.assertEqual(
+            self.harness.charm.seen_events,
+            ['RadosGWUserEvent'])
+        self.harness.update_relation_data(
+            rel_id2,
+            'ceph-us',
+            {
+                'access-key': 'JGHKJGDKJGJGJHGYYYYM',
+                'secret-key': 'iljkdfhHKHKd88LKxNLSKDiijfjfjfldjfjlf44',
+                'uid': 'radosgw-user-10'})
+        self.assertEqual(
+            self.harness.charm.radosgw_user.get_user_creds(),
+            [
+                {
+                    'access_key': 'XNUZVPL364U0BL1OXWJZ',
+                    'daemon_id': 'juju-80416c-zaza-7af97ef8a776-3',
+                    'secret_key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                    'uid': 'radosgw-user-9'},
+                {
+                    'access_key': 'XNUZVPL364U0BL1OXWJZ',
+                    'daemon_id': 'juju-80416c-zaza-7af97ef8a776-4',
+                    'secret_key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                    'uid': 'radosgw-user-9'},
+                {
+                    'access_key': 'JGHKJGDKJGJGJHGYYYYM',
+                    'daemon_id': 'juju-dddddd-zaza-sdfsfsfs-4',
+                    'secret_key': 'iljkdfhHKHKd88LKxNLSKDiijfjfjfldjfjlf44',
+                    'uid': 'radosgw-user-10'},
+                {
+                    'access_key': 'JGHKJGDKJGJGJHGYYYYM',
+                    'daemon_id': 'juju-dddddd-zaza-sdfsfsfs-5',
+                    'secret_key': 'iljkdfhHKHKd88LKxNLSKDiijfjfjfldjfjlf44',
+                    'uid': 'radosgw-user-10'}])
+
+    def test_add_radosgw_dashboard_relation_missing_data(self):
+        rel_id1 = self.harness.add_relation('radosgw-dashboard', 'ceph-eu')
+        self.harness.begin()
+        self.assertEqual(
+            self.harness.charm.seen_events,
+            [])
+        self.harness.set_leader()
+        self.harness.add_relation_unit(
+            rel_id1,
+            'ceph-eu/0')
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu/0',
+            {
+                'daemon-id': 'juju-80416c-zaza-7af97ef8a776-3'})
+        self.harness.update_relation_data(
+            rel_id1,
+            'ceph-eu',
+            {
+                'secret-key': 'SgBo115xJcW90nkQ5EaNQ6fPeyeUUT0GxhwQbLFo',
+                'uid': 'radosgw-user-9'})
+        self.assertEqual(
+            self.harness.charm.radosgw_user.get_user_creds(),
+            [])
+        self.assertEqual(
+            self.harness.charm.seen_events,
+            [])