Adds support for migration to multi-site system.

1.) Currently multi-site can only be configured when system is being
deployed from scratch, migration works by renaming the existing
Zone/Zonegroups (Z/ZG) to Juju config values on primary site before
secondary site pulls the realm data and then rename and configure
secondary Zone accordingly.

During migration:
2.) If multiple Z/ZG not matching the config values are present at
primary site, the leader unit will block and prompt use of
'force-enable-multisite' which renames and configures selected Z/ZG
according to multisite config values.

3.) If the site being added as a secondary already contain Buckets,
the unit will block and prompt the operator to purge all such Buckets
before proceeding.

Closes-Bug: #1959837
Change-Id: I01a4c1c4551c797f0a32951dfbde8a1a4126c2d6
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/840

actions.yaml

@@ -10,3 +10,12 @@ readwrite:
   description: Mark the zone associated with the local units as read/write (multi-site).
 tidydefaults:
   description: Delete default zone and zonegroup configuration (multi-site).
+force-enable-multisite:
+  description: Reconfigure provided Zone and Zonegroup for migration to multisite.
+  params:
+    zone:
+      type: string
+      description: Existing Zone to be reconfigured as the 'zone' config value.
+    zonegroup:
+      type: string
+      description: Existing Zonegroup to be reconfigured as the 'zonegroup' config value.

actions/actions.py

@@ -17,6 +17,7 @@
 import os
 import subprocess
 import sys
+import uuid
 
 sys.path.append('hooks/')
 
@@ -25,12 +26,27 @@ import multisite
 from charmhelpers.core.hookenv import (
     action_fail,
     config,
+    is_leader,
+    leader_set,
     action_set,
+    action_get,
+    log,
+    ERROR,
+    DEBUG,
+)
+from charmhelpers.contrib.openstack.ip import (
+    canonical_url,
+    PUBLIC,
 )
 from utils import (
     pause_unit_helper,
     resume_unit_helper,
     register_configs,
+    listen_port,
+    service_name,
+)
+from charmhelpers.core.host import (
+    service_restart,
 )
 
 
@@ -50,13 +66,19 @@ def resume(args):
 def promote(args):
     """Promote zone associated with local RGW units to master/default"""
     zone = config('zone')
+    zonegroup = config('zonegroup')
+    if not is_leader():
+        action_fail('This action can only be executed on leader unit.')
+        return
     if not zone:
         action_fail('No zone configuration set, not promoting')
         return
     try:
         multisite.modify_zone(zone,
                               default=True, master=True)
-        multisite.update_period()
+        multisite.update_period(zonegroup=zonegroup, zone=zone)
+        leader_set(restart_nonce=str(uuid.uuid4()))
+        service_restart(service_name())
         action_set(
             values={'message': 'zone:{} promoted to '
                     'master/default'.format(zone)}
@@ -122,6 +144,89 @@ def tidydefaults(args):
                     ': {} - {}'.format(zone, cpe.output))
 
 
+def force_enable_multisite(args):
+    """Configure provided zone and zonegroup according to Multisite Config
+
+    In a situation when multiple zone or zonegroups are configured on the
+    primary site, the decision for which pair to use in multisite system
+    is taken through this action. It takes provided parameters (zone name
+    and zonegroup name) and rename/ modify them appropriately.
+    """
+    public_url = '{}:{}'.format(
+        canonical_url(register_configs(), PUBLIC),
+        listen_port(),
+    )
+    current_zone = action_get("zone")
+    current_zonegroup = action_get("zonegroup")
+    endpoints = [public_url]
+    realm = config('realm')
+    new_zone = config('zone')
+    new_zonegroup = config('zonegroup')
+
+    log("zone:{}, zonegroup:{}, endpoints:{}, realm:{}, new_zone:{}, "
+        "new_zonegroup:{}".format(
+            current_zone, current_zonegroup, endpoints,
+            realm, new_zone, new_zonegroup
+        ), level=DEBUG)
+
+    if not is_leader():
+        action_fail('This action can only be executed on leader unit.')
+        return
+
+    if not all((realm, new_zonegroup, new_zone)):
+        action_fail("Missing required charm configurations realm({}), "
+                    "zonegroup({}) and zone({}).".format(
+                        realm, new_zonegroup, new_zone
+                    ))
+        return
+
+    if current_zone not in multisite.list_zones():
+        action_fail('Provided zone {} does not exist.'.format(current_zone))
+        return
+
+    if current_zonegroup not in multisite.list_zonegroups():
+        action_fail('Provided zone {} does not exist.'
+                    .format(current_zonegroup))
+        return
+
+    try:
+        # Rename chosen zonegroup/zone as per charm config value.
+        rename_result = multisite.rename_multisite_config(
+            [current_zonegroup],
+            new_zonegroup,
+            [current_zone], new_zone
+        )
+        if rename_result is None:
+            action_fail('Failed to rename zone {} or zonegroup {}.'
+                        .format(current_zone, current_zonegroup))
+            return
+
+        # Configure zonegroup/zone as master for multisite.
+        modify_result = multisite.modify_multisite_config(
+            new_zone, new_zonegroup,
+            realm=realm,
+            endpoints=endpoints
+        )
+        if modify_result is None:
+            action_fail('Failed to configure zone {} or zonegroup {}.'
+                        .format(new_zonegroup, new_zone))
+            return
+
+        leader_set(restart_nonce=str(uuid.uuid4()))
+        service_restart(service_name())
+        action_set(
+            values={
+                'message': 'Multisite Configuration Resolved'
+            }
+        )
+    except subprocess.CalledProcessError as cpe:
+        message = "Failed to configure zone ({}) and zonegroup ({})".format(
+            current_zone, current_zonegroup
+        )
+        log(message, level=ERROR)
+        action_fail(message + " : {}".format(cpe.output))
+
+
 # A dictionary of all the defined actions to callables (which take
 # parsed arguments).
 ACTIONS = {
@@ -131,6 +236,7 @@ ACTIONS = {
     "readonly": readonly,
     "readwrite": readwrite,
     "tidydefaults": tidydefaults,
+    "force-enable-multisite": force_enable_multisite,
 }
 
 

actions/force-enable-multisite

@@ -0,0 +1 @@
+actions.py

hooks/ceph_radosgw_context.py

@@ -305,7 +305,7 @@ class MonContext(context.CephContext):
         ctxt.update(user_provided)
 
         if self.context_complete(ctxt):
-            # Multi-site Zone configuration is optional,
+            # Multi-site zone configuration is optional,
             # so add after assessment
             ctxt['rgw_zone'] = config('zone')
             ctxt['rgw_zonegroup'] = config('zonegroup')

hooks/hooks.py

@@ -27,6 +27,7 @@ import charms_ceph.utils as ceph_utils
 import multisite
 
 from charmhelpers.core.hookenv import (
+    ERROR,
     relation_get,
     relation_id as ch_relation_id,
     relation_ids,
@@ -366,7 +367,7 @@ def mon_relation(rid=None, unit=None):
                 existing_zones = multisite.list_zones()
                 log('Existing zones {}'.format(existing_zones), level=DEBUG)
                 if zone not in existing_zones:
-                    log("Zone '{}' doesn't exist, creating".format(zone))
+                    log("zone '{}' doesn't exist, creating".format(zone))
                     try:
                         multisite.create_zone(zone,
                                               endpoints=endpoints,
@@ -377,7 +378,7 @@ def mon_relation(rid=None, unit=None):
                             # NOTE(lourot): may have been created in the
                             # background by the Rados Gateway daemon, see
                             # lp:1856106
-                            log("Zone '{}' existed already after all".format(
+                            log("zone '{}' existed already after all".format(
                                 zone))
                         else:
                             raise
@@ -741,8 +742,43 @@ def master_relation_joined(relation_id=None):
         multisite.create_realm(realm, default=True)
         mutation = True
 
+    # Migration if master site has buckets configured.
+    # Migration involves renaming existing zone/zongroups such that existing
+    # buckets and their objects can be preserved on the master site.
+    if multisite.check_cluster_has_buckets() is True:
+        log('Migrating to multisite with zone ({}) and zonegroup ({})'
+            .format(zone, zonegroup), level=DEBUG)
+        zones = multisite.list_zones()
+        zonegroups = multisite.list_zonegroups()
+
+        if (len(zonegroups) > 1) and (zonegroup not in zonegroups):
+            log('Multiple zonegroups found {}, aborting.'
+                .format(zonegroups), level=ERROR)
+            return
+
+        if (len(zones) > 1) and (zone not in zones):
+            log('Multiple zones found {}, aborting.'
+                .format(zones), level=ERROR)
+            return
+
+        rename_result = multisite.rename_multisite_config(
+            zonegroups, zonegroup,
+            zones, zone
+        )
+        if rename_result is None:
+            return
+
+        modify_result = multisite.modify_multisite_config(
+            zone, zonegroup,
+            endpoints=endpoints,
+            realm=realm
+        )
+        if modify_result is None:
+            return
+        mutation = True
+
     if zonegroup not in multisite.list_zonegroups():
-        log('Zonegroup {} not found, creating now'.format(zonegroup))
+        log('zonegroup {} not found, creating now'.format(zonegroup))
         multisite.create_zonegroup(zonegroup,
                                    endpoints=endpoints,
                                    default=True, master=True,
@@ -750,7 +786,7 @@ def master_relation_joined(relation_id=None):
         mutation = True
 
     if zone not in multisite.list_zones():
-        log('Zone {} not found, creating now'.format(zone))
+        log('zone {} not found, creating now'.format(zone))
         multisite.create_zone(zone,
                               endpoints=endpoints,
                               default=True, master=True,
@@ -773,7 +809,7 @@ def master_relation_joined(relation_id=None):
         log(
             'Mutation detected. Restarting {}.'.format(service_name()),
             'INFO')
-        multisite.update_period()
+        multisite.update_period(zonegroup=zonegroup, zone=zone)
         service_restart(service_name())
         leader_set(restart_nonce=str(uuid.uuid4()))
     else:
@@ -829,6 +865,13 @@ def slave_relation_changed(relation_id=None, unit=None):
 
     mutation = False
 
+    # NOTE(utkarshbhatthere):
+    # A site with existing data can create inconsistencies when added as a
+    # secondary site for RGW. Hence it must be pristine.
+    if multisite.check_cluster_has_buckets():
+        log("Non-Pristine site can't be used as secondary", level=ERROR)
+        return
+
     if realm not in multisite.list_realms():
         log('Realm {} not found, pulling now'.format(realm))
         multisite.pull_realm(url=master_data['url'],
@@ -841,7 +884,7 @@ def slave_relation_changed(relation_id=None, unit=None):
         mutation = True
 
     if zone not in multisite.list_zones():
-        log('Zone {} not found, creating now'.format(zone))
+        log('zone {} not found, creating now'.format(zone))
         multisite.create_zone(zone,
                               endpoints=endpoints,
                               default=False, master=False,
@@ -854,7 +897,7 @@ def slave_relation_changed(relation_id=None, unit=None):
         log(
             'Mutation detected. Restarting {}.'.format(service_name()),
             'INFO')
-        multisite.update_period()
+        multisite.update_period(zonegroup=zonegroup, zone=zone)
         service_restart(service_name())
         leader_set(restart_nonce=str(uuid.uuid4()))
     else:

hooks/multisite.py

@@ -25,7 +25,7 @@ import charmhelpers.core.decorators as decorators
 RGW_ADMIN = 'radosgw-admin'
 
 
-@decorators.retry_on_exception(num_retries=5, base_delay=3,
+@decorators.retry_on_exception(num_retries=10, base_delay=5,
                                exc_type=subprocess.CalledProcessError)
 def _check_output(cmd):
     """Logging wrapper for subprocess.check_ouput"""
@@ -105,6 +105,32 @@ list_zonegroups = functools.partial(_list, 'zonegroup')
 list_users = functools.partial(_list, 'user')
 
 
+def list_buckets(zone, zonegroup):
+    """List Buckets served under the provided zone and zonegroup pair.
+
+    :param zonegroup: Parent zonegroup.
+    :type zonegroup: str
+    :param zone: Parent zone.
+    :type zone: str
+    :returns: List of buckets found
+    :rtype: list
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'bucket', 'list',
+        '--rgw-zone={}'.format(zone),
+        '--rgw-zonegroup={}'.format(zonegroup),
+    ]
+    try:
+        return json.loads(_check_output(cmd))
+    except subprocess.CalledProcessError:
+        hookenv.log("Bucket queried for incorrect zone({})-zonegroup({}) "
+                    "pair".format(zone, zonegroup), level=hookenv.ERROR)
+        return None
+    except TypeError:
+        return None
+
+
 def create_realm(name, default=False):
     """
     Create a new RADOS Gateway Realm.
@@ -146,7 +172,7 @@ def set_default_realm(name):
 
 def create_zonegroup(name, endpoints, default=False, master=False, realm=None):
     """
-    Create a new RADOS Gateway Zone Group
+    Create a new RADOS Gateway zone Group
 
     :param name: name of zonegroup to create
     :type name: str
@@ -179,10 +205,49 @@ def create_zonegroup(name, endpoints, default=False, master=False, realm=None):
         return None
 
 
+def modify_zonegroup(name, endpoints=None, default=False,
+                     master=False, realm=None):
+    """Modify an existing RADOS Gateway zonegroup
+
+    An empty list of endpoints would cause NO-CHANGE in the configured
+    endpoints for the zonegroup.
+
+    :param name: name of zonegroup to modify
+    :type name: str
+    :param endpoints: list of URLs to endpoints for zonegroup
+    :type endpoints: list[str]
+    :param default: set zonegroup as the default zonegroup
+    :type default: boolean
+    :param master: set zonegroup as the master zonegroup
+    :type master: boolean
+    :param realm: realm name for provided zonegroup
+    :type realm: str
+    :return: zonegroup configuration
+    :rtype: dict
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'zonegroup', 'modify',
+        '--rgw-zonegroup={}'.format(name),
+    ]
+    if realm:
+        cmd.append('--rgw-realm={}'.format(realm))
+    if endpoints:
+        cmd.append('--endpoints={}'.format(','.join(endpoints)))
+    if default:
+        cmd.append('--default')
+    if master:
+        cmd.append('--master')
+    try:
+        return json.loads(_check_output(cmd))
+    except TypeError:
+        return None
+
+
 def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
                 access_key=None, secret=None, readonly=False):
     """
-    Create a new RADOS Gateway Zone
+    Create a new RADOS Gateway zone
 
     :param name: name of zone to create
     :type name: str
@@ -226,9 +291,9 @@ def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
 
 
 def modify_zone(name, endpoints=None, default=False, master=False,
-                access_key=None, secret=None, readonly=False):
+                access_key=None, secret=None, readonly=False,
-    """
+                realm=None, zonegroup=None):
-    Modify an existing RADOS Gateway zone
+    """Modify an existing RADOS Gateway zone
 
     :param name: name of zone to create
     :type name: str
@@ -243,7 +308,11 @@ def modify_zone(name, endpoints=None, default=False, master=False,
     :param secret: secret to use with access-key for the zone
     :type secret: str
     :param readonly: set zone as read only
-    :type: readonly: boolean
+    :type readonly: boolean
+    :param realm: realm to use for zone
+    :type realm: str
+    :param zonegroup: zonegroup to use for zone
+    :type zonegroup: str
     :return: zone configuration
     :rtype: dict
     """
@@ -252,6 +321,10 @@ def modify_zone(name, endpoints=None, default=False, master=False,
         'zone', 'modify',
         '--rgw-zone={}'.format(name),
     ]
+    if realm:
+        cmd.append('--rgw-realm={}'.format(realm))
+    if zonegroup:
+        cmd.append('--rgw-zonegroup={}'.format(zonegroup))
     if endpoints:
         cmd.append('--endpoints={}'.format(','.join(endpoints)))
     if access_key and secret:
@@ -268,14 +341,24 @@ def modify_zone(name, endpoints=None, default=False, master=False,
         return None
 
 
-def update_period(fatal=True):
+def update_period(fatal=True, zonegroup=None, zone=None):
-    """
+    """Update RADOS Gateway configuration period
-    Update RADOS Gateway configuration period
+
+    :param fatal: In failure case, whether CalledProcessError is to be raised.
+    :type fatal: boolean
+    :param zonegroup: zonegroup name
+    :type zonegroup: str
+    :param zone: zone name
+    :type zone: str
     """
     cmd = [
         RGW_ADMIN, '--id={}'.format(_key_name()),
         'period', 'update', '--commit'
     ]
+    if zonegroup is not None:
+        cmd.append('--rgw-zonegroup={}'.format(zonegroup))
+    if zone is not None:
+        cmd.append('--rgw-zone={}'.format(zone))
     if fatal:
         _check_call(cmd)
     else:
@@ -439,3 +522,279 @@ def pull_period(url, access_key, secret):
         return json.loads(_check_output(cmd))
     except TypeError:
         return None
+
+
+def rename_zone(name, new_name, zonegroup):
+    """Rename an existing RADOS Gateway zone
+
+    If the command execution succeeds, 0 is returned, otherwise
+    None is returned to the caller.
+
+    :param name: current name for the zone being renamed
+    :type name: str
+    :param new_name: new name for the zone being renamed
+    :type new_name: str
+    :rtype: int
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'zone', 'rename',
+        '--rgw-zone={}'.format(name),
+        '--zone-new-name={}'.format(new_name),
+        '--rgw-zonegroup={}'.format(zonegroup)
+    ]
+    result = _call(cmd)
+    return 0 if result == 0 else None
+
+
+def rename_zonegroup(name, new_name):
+    """Rename an existing RADOS Gateway zonegroup
+
+    If the command execution succeeds, 0 is returned, otherwise
+    None is returned to the caller.
+
+    :param name: current name for the zonegroup being renamed
+    :type name: str
+    :param new_name: new name for the zonegroup being renamed
+    :type new_name: str
+    :rtype: int
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'zonegroup', 'rename',
+        '--rgw-zonegroup={}'.format(name),
+        '--zonegroup-new-name={}'.format(new_name),
+    ]
+    result = _call(cmd)
+    return 0 if result == 0 else None
+
+
+def get_zonegroup_info(zonegroup):
+    """Fetch detailed info for the provided zonegroup
+
+    :param zonegroup: zonegroup Name for detailed query
+    :type zonegroup: str
+    :rtype: dict
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'zonegroup', 'get',
+        '--rgw-zonegroup={}'.format(zonegroup),
+    ]
+    try:
+        return json.loads(_check_output(cmd))
+    except TypeError:
+        return None
+
+
+def get_sync_status():
+    """
+    Get sync status
+    :returns: Sync Status Report from radosgw-admin
+    :rtype: str
+    """
+    cmd = [
+        RGW_ADMIN, '--id={}'.format(_key_name()),
+        'sync', 'status',
+    ]
+    try:
+        return _check_output(cmd)
+    except subprocess.CalledProcessError:
+        hookenv.log("Failed to fetch sync status", level=hookenv.ERROR)
+        return None
+
+
+def is_multisite_configured(zone, zonegroup):
+    """Check if system is already multisite configured
+
+    Checks if zone and zonegroup are configured appropriately and
+    remote data sync source is detected in sync status
+
+    :rtype: Boolean
+    """
+    if zone not in list_zones():
+        hookenv.log("No local zone found with name ({})".format(zonegroup),
+                    level=hookenv.ERROR)
+        return False
+
+    if zonegroup not in list_zonegroups():
+        hookenv.log("No zonegroup found with name ({})".format(zonegroup),
+                    level=hookenv.ERROR)
+        return False
+
+    sync_status = get_sync_status()
+    if sync_status is not None:
+        return ('data sync source:' in sync_status)
+
+    return False
+
+
+def get_local_zone(zonegroup):
+    """Get local zone to provided parent zonegroup.
+
+    In multisite systems, zonegroup contains both local and remote zone info
+    this method is used to fetch the zone local to querying site.
+
+    :param zonegroup: parent zonegroup name.
+    :type zonegroup: str
+    :returns: tuple with parent zonegroup and local zone name
+    :rtype: tuple
+    """
+    local_zones = list_zones()
+    zonegroup_info = get_zonegroup_info(zonegroup)
+
+    if zonegroup_info is None:
+        hookenv.log("Failed to fetch zonegroup ({}) info".format(zonegroup),
+                    level=hookenv.ERROR)
+        return None
+
+    # zonegroup info always contains self name and zones list so fetching
+    # directly is safe.
+    master_zonegroup = zonegroup_info['name']
+    for zone_info in zonegroup_info['zones']:
+        zone = zone_info['name']
+        if zone in local_zones:
+            return zone, master_zonegroup
+
+    hookenv.log(
+        "No local zone configured for zonegroup ({})".format(zonegroup),
+        level=hookenv.ERROR
+    )
+    return None
+
+
+def rename_multisite_config(zonegroups, new_zonegroup_name,
+                            zones, new_zone_name):
+    """Rename zone and zonegroup to provided new names.
+
+    If zone list (zones) or zonegroup list (zonegroups) contain 1 element
+    rename the only element present in the list to provided (new_) value.
+
+    :param zonegroups: List of zonegroups available at site.
+    :type zonegroups: list[str]
+    :param new_zonegroup_name: Desired new name for master zonegroup.
+    :type new_zonegroup_name: str
+    :param zones: List of zones available at site.
+    :type zones: list[str]
+    :param new_zonegroup_name: Desired new name for master zone.
+    :type new_zonegroup_name: str
+
+    :return: Whether any of the zone or zonegroup is renamed.
+    :rtype: Boolean
+    """
+    mutation = False
+    if (len(zonegroups) == 1) and (len(zones) == 1):
+        if new_zonegroup_name not in zonegroups:
+            result = rename_zonegroup(zonegroups[0], new_zonegroup_name)
+            if result is None:
+                hookenv.log(
+                    "Failed renaming zonegroup from {} to {}"
+                    .format(zonegroups[0], new_zonegroup_name),
+                    level=hookenv.ERROR
+                )
+                return None
+            mutation = True
+
+        if new_zone_name not in zones:
+            result = rename_zone(zones[0], new_zone_name, new_zonegroup_name)
+            if result is None:
+                hookenv.log(
+                    "Failed renaming zone from {} to {}"
+                    .format(zones[0], new_zone_name), level=hookenv.ERROR
+                )
+                return None
+            mutation = True
+
+    if mutation:
+        hookenv.log("Renamed zonegroup {} to {}, and zone {} to {}".format(
+                    zonegroups[0], new_zonegroup_name,
+                    zones[0], new_zone_name))
+        return True
+
+    return False
+
+
+def modify_multisite_config(zone, zonegroup, endpoints=None, realm=None):
+    """Configure zone and zonegroup as master for multisite system.
+
+    :param zonegroup: zonegroup name being configured for multisite
+    :type zonegroup: str
+    :param zone: zone name being configured for multisite
+    :type zone: str
+    :param endpoints: list of URLs to RGW endpoints
+    :type endpoints: list[str]
+    :param realm: realm to use for multisite
+    :type realm: str
+    :rtype: Boolean
+    """
+    if modify_zonegroup(zonegroup, endpoints=endpoints, default=True,
+                        master=True, realm=realm) is None:
+        hookenv.log(
+            "Failed configuring zonegroup {}".format(zonegroup),
+            level=hookenv.ERROR
+        )
+        return None
+
+    if modify_zone(zone, endpoints=endpoints, default=True,
+                   master=True, zonegroup=zonegroup, realm=realm) is None:
+        hookenv.log(
+            "Failed configuring zone {}".format(zone), level=hookenv.ERROR
+        )
+        return None
+
+    update_period(zonegroup=zonegroup, zone=zone)
+    hookenv.log("Configured zonegroup {}, and zone {} for multisite".format(
+                zonegroup, zone))
+    return True
+
+
+def check_zone_has_buckets(zone, zonegroup):
+    """Checks whether provided zone-zonegroup pair contains any bucket.
+
+    :param zone: zone name to query buckets in.
+    :type zone: str
+    :param zonegroup: Parent zonegroup of zone.
+    :type zonegroup: str
+    :rtype: Boolean
+    """
+    buckets = list_buckets(zone, zonegroup)
+    if buckets is not None:
+        return (len(buckets) > 0)
+    hookenv.log(
+        "Failed to query buckets for zone {} zonegroup {}"
+        .format(zone, zonegroup),
+        level=hookenv.WARNING
+    )
+    return False
+
+
+def check_zonegroup_has_buckets(zonegroup):
+    """Checks whether any bucket exists in the master zone of a zonegroup.
+
+    :param zone: zonegroup name to query buckets.
+    :type zone: str
+    :rtype: Boolean
+    """
+    # NOTE(utkarshbhatthere): sometimes querying against a particular
+    # zonegroup results in info of an entirely different zonegroup, thus to
+    # prevent a query against an incorrect pair in such cases, both zone and
+    # zonegroup names are taken from zonegroup info.
+    master_zone, master_zonegroup = get_local_zone(zonegroup)
+
+    # If master zone is not configured for zonegroup
+    if master_zone is None:
+        hookenv.log("No master zone configured for zonegroup {}"
+                    .format(master_zonegroup), level=hookenv.WARNING)
+        return False
+    return check_zone_has_buckets(master_zone, master_zonegroup)
+
+
+def check_cluster_has_buckets():
+    """Iteratively check if ANY zonegroup has buckets on cluster.
+
+    :rtype: Boolean
+    """
+    for zonegroup in list_zonegroups():
+        if check_zonegroup_has_buckets(zonegroup):
+            return True
+    return False

hooks/utils.py

@@ -20,6 +20,7 @@ from collections import OrderedDict
 from copy import deepcopy
 
 import ceph_radosgw_context
+import multisite
 
 from charmhelpers.core.hookenv import (
     relation_get,
@@ -184,6 +185,14 @@ def get_optional_interfaces():
     return optional_interfaces
 
 
+def get_zones_zonegroups():
+    """Get a tuple with lists of zones and zonegroups existing on site
+
+    :rtype: tuple
+    """
+    return multisite.list_zones(), multisite.list_zonegroups()
+
+
 def check_optional_config_and_relations(configs):
     """Check that if we have a relation_id for high availability that we can
     get the hacluster config.  If we can't then we are blocked.  This function
@@ -201,22 +210,63 @@ def check_optional_config_and_relations(configs):
             return ('blocked',
                     'hacluster missing configuration: '
                     'vip, vip_iface, vip_cidr')
-    # NOTE: misc multi-site relation and config checks
+
     multisite_config = (config('realm'),
                         config('zonegroup'),
                         config('zone'))
-    if relation_ids('master') or relation_ids('slave'):
+    master_configured = (leader_get('access_key'),
+                         leader_get('secret'),
+                         leader_get('restart_nonce'))
+
+    # Any realm or zonegroup config is present, multisite checks can be done.
+    if (config('realm') or config('zonegroup')):
+        # All of Realm, zonegroup, and zone must be configured.
         if not all(multisite_config):
             return ('blocked',
                     'multi-site configuration incomplete '
                     '(realm={realm}, zonegroup={zonegroup}'
                     ', zone={zone})'.format(**config()))
-    if (all(multisite_config) and not
+
-            (relation_ids('master') or relation_ids('slave'))):
+        # Master/Slave Relation should be configured.
+        if not (relation_ids('master') or relation_ids('slave')):
             return ('blocked',
                     'multi-site configuration but master/slave '
                     'relation missing')
-    if (all(multisite_config) and relation_ids('slave')):
+
+        # Primary site status check
+        if relation_ids('master'):
+            # Migration: The system is not multisite already.
+            if not multisite.is_multisite_configured(config('zone'),
+                                                     config('zonegroup')):
+                if multisite.check_cluster_has_buckets():
+                    zones, zonegroups = get_zones_zonegroups()
+                    status_msg = "Multiple zone or zonegroup configured, " \
+                                 "use action 'config-multisite-values' to " \
+                                 "resolve."
+                    if (len(zonegroups) > 1 and
+                            config('zonegroup') not in zonegroups):
+                        return('blocked', status_msg)
+
+                    if len(zones) > 1 and config('zone') not in zones:
+                        return('blocked', status_msg)
+
+                    if not all(master_configured):
+                        return ('blocked', "Failure in Multisite migration, "
+                                           "Refer to Logs.")
+            # Non-Migration scenario.
+            if not all(master_configured):
+                return ('waiting',
+                        'waiting for configuration of master zone')
+
+        # Secondary site status check
+        if relation_ids('slave'):
+            # Migration: The system is not multisite already.
+            if not multisite.is_multisite_configured(config('zone'),
+                                                     config('zonegroup')):
+                if multisite.check_cluster_has_buckets():
+                    return ('blocked',
+                            "Non-Pristine RGW site can't be used as secondary")
+
             multisite_ready = False
             for rid in relation_ids('slave'):
                 for unit in related_units(rid):
@@ -226,16 +276,6 @@ def check_optional_config_and_relations(configs):
             if not multisite_ready:
                 return ('waiting',
                         'multi-site master relation incomplete')
-    master_configured = (
-        leader_get('access_key'),
-        leader_get('secret'),
-        leader_get('restart_nonce'),
-    )
-    if (all(multisite_config) and
-            relation_ids('master') and
-            not all(master_configured)):
-        return ('waiting',
-                'waiting for configuration of master zone')
 
     # Check that provided Ceph BlueStoe configuration is valid.
     try:

osci.yaml

@@ -4,12 +4,17 @@
       - charm-unit-jobs-py39
     check:
       jobs:
+        - focal-xena-multisite
         - vault-focal-xena_rgw
         - vault-focal-xena-namespaced
+        - focal-yoga-multisite:
+            voting: false
         - vault-focal-yoga_rgw:
             voting: false
         - vault-focal-yoga-namespaced:
             voting: false
+        - jammy-yoga-multisite:
+            voting: false
         - vault-jammy-yoga_rgw:
             voting: false
         - vault-jammy-yoga-namespaced:
@@ -18,6 +23,16 @@
       needs_charm_build: true
       charm_build_name: ceph-radosgw
       build_type: charmcraft
+- job:
+    name: focal-xena-multisite
+    parent: func-target
+    dependencies:
+      - osci-lint
+      - charm-build
+      - tox-py38
+      - tox-py39
+    vars:
+      tox_extra_args: focal-xena-multisite
 - job:
     name: vault-focal-xena_rgw
     parent: func-target
@@ -38,6 +53,13 @@
     vars:
       tox_extra_args: vault:focal-xena-namespaced
 
+- job:
+    name: jammy-yoga-multisite
+    parent: func-target
+    dependencies:
+      - focal-xena-multisite
+    vars:
+      tox_extra_args: jammy-yoga-multisite
 - job:
     name: vault-jammy-yoga_rgw
     parent: func-target
@@ -54,6 +76,13 @@
       - vault-focal-xena-namespaced
     vars:
       tox_extra_args: vault:jammy-yoga-namespaced
+- job:
+    name: focal-yoga-multisite
+    parent: func-target
+    dependencies:
+      - focal-xena-multisite
+    vars:
+      tox_extra_args: focal-yoga-multisite
 - job:
     name: vault-focal-yoga_rgw
     parent: func-target

tests/bundles/focal-xena-multisite.yaml

@@ -0,0 +1,98 @@
+options:
+  source: &source cloud:focal-xena
+
+series: focal
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+  '1':
+  '2':
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+
+applications:
+  ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '0'
+
+  secondary-ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '1'
+
+  ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '2'
+      - '6'
+      - '7'
+    channel: latest/edge
+
+  secondary-ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '3'
+      - '8'
+      - '9'
+    channel: latest/edge
+
+  ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '4'
+    channel: latest/edge
+
+  secondary-ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '5'
+    channel: latest/edge
+
+relations:
+  - - 'ceph-osd:mon'
+    - 'ceph-mon:osd'
+
+  - - 'ceph-radosgw:mon'
+    - 'ceph-mon:radosgw'
+
+  - - 'secondary-ceph-osd:mon'
+    - 'secondary-ceph-mon:osd'
+
+  - - 'secondary-ceph-radosgw:mon'
+    - 'secondary-ceph-mon:radosgw'

tests/bundles/focal-yoga-multisite.yaml

@@ -0,0 +1,99 @@
+options:
+  source: &source cloud:focal-yoga
+
+series: focal
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+  '1':
+  '2':
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+
+applications:
+  ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '0'
+
+  secondary-ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '1'
+
+  ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '2'
+      - '6'
+      - '7'
+    channel: latest/edge
+
+  secondary-ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '3'
+      - '8'
+      - '9'
+    channel: latest/edge
+
+  ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '4'
+    channel: latest/edge
+
+  secondary-ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '5'
+    channel: latest/edge
+
+relations:
+  - - 'ceph-osd:mon'
+    - 'ceph-mon:osd'
+
+  - - 'ceph-radosgw:mon'
+    - 'ceph-mon:radosgw'
+
+  - - 'secondary-ceph-osd:mon'
+    - 'secondary-ceph-mon:osd'
+
+  - - 'secondary-ceph-radosgw:mon'
+    - 'secondary-ceph-mon:radosgw'
+

tests/bundles/jammy-yoga-multisite.yaml

@@ -0,0 +1,99 @@
+options:
+  source: &source distro
+
+series: jammy
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+  '1':
+  '2':
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+
+applications:
+  ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '0'
+
+  secondary-ceph-radosgw:
+    charm: ../../ceph-radosgw.charm
+    num_units: 1
+    options:
+      source: *source
+    to:
+      - '1'
+
+  ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '2'
+      - '6'
+      - '7'
+    channel: latest/edge
+
+  secondary-ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    constraints: "mem=2048"
+    storage:
+      osd-devices: 'cinder,10G'
+    options:
+      source: *source
+      osd-devices: '/srv/ceph /dev/test-non-existent'
+    to:
+      - '3'
+      - '8'
+      - '9'
+    channel: latest/edge
+
+  ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '4'
+    channel: latest/edge
+
+  secondary-ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 1
+    options:
+      monitor-count: 1
+      source: *source
+    to:
+      - '5'
+    channel: latest/edge
+
+relations:
+  - - 'ceph-osd:mon'
+    - 'ceph-mon:osd'
+
+  - - 'ceph-radosgw:mon'
+    - 'ceph-mon:radosgw'
+
+  - - 'secondary-ceph-osd:mon'
+    - 'secondary-ceph-mon:osd'
+
+  - - 'secondary-ceph-radosgw:mon'
+    - 'secondary-ceph-mon:radosgw'
+

tests/tests.yaml

@@ -1,13 +1,17 @@
 charm_name: ceph-radosgw
 
 gate_bundles:
+  - focal-xena-multisite
   - vault: focal-xena
   - vault: focal-xena-namespaced
 
 smoke_bundles:
+  - focal-xena-multisite
   - vault: focal-xena
 
 dev_bundles:
+  - focal-yoga-multisite
+  - jammy-yoga-multisite
   - vault: focal-yoga
   - vault: focal-yoga-namespaced
   - vault: jammy-yoga
@@ -16,7 +20,7 @@ dev_bundles:
 target_deploy_status:
   vault:
     workload-status: blocked
-    workload-status-message: Vault needs to be initialized
+    workload-status-message-prefix: Vault needs to be initialized
 
 configure:
   - vault:

unit_tests/test_actions.py

@@ -85,6 +85,9 @@ class MultisiteActionsTestCase(CharmTestCase):
         'action_set',
         'multisite',
         'config',
+        'is_leader',
+        'leader_set',
+        'service_name',
     ]
 
     def setUp(self):
@@ -93,17 +96,23 @@ class MultisiteActionsTestCase(CharmTestCase):
         self.config.side_effect = self.test_config.get
 
     def test_promote(self):
+        self.is_leader.return_value = True
         self.test_config.set('zone', 'testzone')
+        self.test_config.set('zonegroup', 'testzonegroup')
         actions.promote([])
         self.multisite.modify_zone.assert_called_once_with(
             'testzone',
             default=True,
             master=True,
         )
-        self.multisite.update_period.assert_called_once_with()
+        self.multisite.update_period.assert_called_once_with(
+            zonegroup='testzonegroup', zone='testzone'
+        )
 
     def test_promote_unconfigured(self):
+        self.is_leader.return_value = True
         self.test_config.set('zone', None)
+        self.test_config.set('zonegroup', None)
         actions.promote([])
         self.action_fail.assert_called_once()
 

unit_tests/test_hooks.py

@@ -740,7 +740,7 @@ class MasterMultisiteTests(CephRadosMultisiteTests):
         )
         self.multisite.update_period.assert_has_calls([
             call(fatal=False),
-            call(),
+            call(zonegroup='testzonegroup', zone='testzone'),
         ])
         self.service_restart.assert_called_once_with('rgw@hostname')
         self.leader_set.assert_has_calls([
@@ -827,6 +827,7 @@ class SlaveMultisiteTests(CephRadosMultisiteTests):
         self.relation_get.return_value = self._test_relation
         self.multisite.list_realms.return_value = []
         self.multisite.list_zones.return_value = []
+        self.multisite.check_cluster_has_buckets.return_value = False
         ceph_hooks.slave_relation_changed('slave:1', 'rgw/0')
         self.config.assert_has_calls([
             call('realm'),
@@ -857,7 +858,7 @@ class SlaveMultisiteTests(CephRadosMultisiteTests):
         )
         self.multisite.update_period.assert_has_calls([
             call(fatal=False),
-            call(),
+            call(zonegroup='testzonegroup', zone='testzone2'),
         ])
         self.service_restart.assert_called_once()
         self.leader_set.assert_called_once_with(restart_nonce=ANY)

unit_tests/test_multisite.py

@@ -25,6 +25,19 @@ def whoami():
     return inspect.stack()[1][3]
 
 
+def get_zonegroup_stub():
+    # populate dummy zone info
+    zone = {}
+    zone['id'] = "test_zone_id"
+    zone['name'] = "test_zone"
+
+    # populate dummy zonegroup info
+    zonegroup = {}
+    zonegroup['name'] = "test_zonegroup"
+    zonegroup['zones'] = [zone]
+    return zonegroup
+
+
 class TestMultisiteHelpers(CharmTestCase):
 
     TO_PATCH = [
@@ -285,3 +298,159 @@ class TestMultisiteHelpers(CharmTestCase):
             '--url=http://master:80',
             '--access-key=testkey', '--secret=testsecret',
         ], stderr=mock.ANY)
+
+    def test_list_buckets(self):
+        self.subprocess.CalledProcessError = BaseException
+        multisite.list_buckets('default', 'default')
+        self.subprocess.check_output.assert_called_once_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'bucket', 'list', '--rgw-zone=default',
+            '--rgw-zonegroup=default'
+        ], stderr=mock.ANY)
+
+    def test_rename_zonegroup(self):
+        multisite.rename_zonegroup('default', 'test_zone_group')
+        self.subprocess.call.assert_called_once_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zonegroup', 'rename', '--rgw-zonegroup=default',
+            '--zonegroup-new-name=test_zone_group'
+        ])
+
+    def test_rename_zone(self):
+        multisite.rename_zone('default', 'test_zone', 'test_zone_group')
+        self.subprocess.call.assert_called_once_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zone', 'rename', '--rgw-zone=default',
+            '--zone-new-name=test_zone',
+            '--rgw-zonegroup=test_zone_group'
+        ])
+
+    def test_get_zonegroup(self):
+        multisite.get_zonegroup_info('test_zone')
+        self.subprocess.check_output.assert_called_once_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zonegroup', 'get', '--rgw-zonegroup=test_zone'
+        ], stderr=mock.ANY)
+
+    def test_modify_zonegroup_migrate(self):
+        multisite.modify_zonegroup('test_zonegroup',
+                                   endpoints=['http://localhost:80'],
+                                   default=True, master=True,
+                                   realm='test_realm')
+        self.subprocess.check_output.assert_called_once_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zonegroup', 'modify',
+            '--rgw-zonegroup=test_zonegroup', '--rgw-realm=test_realm',
+            '--endpoints=http://localhost:80', '--default', '--master',
+        ], stderr=mock.ANY)
+
+    def test_modify_zone_migrate(self):
+        multisite.modify_zone('test_zone', default=True, master=True,
+                              endpoints=['http://localhost:80'],
+                              zonegroup='test_zonegroup', realm='test_realm')
+        self.subprocess.check_output.assert_called_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zone', 'modify',
+            '--rgw-zone=test_zone', '--rgw-realm=test_realm',
+            '--rgw-zonegroup=test_zonegroup',
+            '--endpoints=http://localhost:80',
+            '--master', '--default', '--read-only=0',
+        ], stderr=mock.ANY)
+
+    @mock.patch.object(multisite, 'list_zones')
+    @mock.patch.object(multisite, 'get_zonegroup_info')
+    def test_get_local_zone(self, mock_get_zonegroup_info, mock_list_zones):
+        mock_get_zonegroup_info.return_value = get_zonegroup_stub()
+        mock_list_zones.return_value = ['test_zone']
+        zone, _zonegroup = multisite.get_local_zone('test_zonegroup')
+        self.assertEqual(
+            zone,
+            'test_zone'
+        )
+
+    def test_rename_multisite_config_zonegroup_fail(self):
+        self.assertEqual(
+            multisite.rename_multisite_config(
+                ['default'], 'test_zonegroup',
+                ['default'], 'test_zone'
+            ),
+            None
+        )
+
+        self.subprocess.call.assert_called_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zonegroup', 'rename', '--rgw-zonegroup=default',
+            '--zonegroup-new-name=test_zonegroup'
+        ])
+
+    def test_modify_multisite_config_zonegroup_fail(self):
+        self.assertEqual(
+            multisite.modify_multisite_config(
+                'test_zone', 'test_zonegroup',
+                endpoints=['http://localhost:80'],
+                realm='test_realm'
+            ),
+            None
+        )
+
+        self.subprocess.check_output.assert_called_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zonegroup', 'modify', '--rgw-zonegroup=test_zonegroup',
+            '--rgw-realm=test_realm',
+            '--endpoints=http://localhost:80', '--default',
+            '--master',
+        ], stderr=mock.ANY)
+
+    @mock.patch.object(multisite, 'modify_zonegroup')
+    def test_modify_multisite_config_zone_fail(self, mock_modify_zonegroup):
+        mock_modify_zonegroup.return_value = True
+        self.assertEqual(
+            multisite.modify_multisite_config(
+                'test_zone', 'test_zonegroup',
+                endpoints=['http://localhost:80'],
+                realm='test_realm'
+            ),
+            None
+        )
+
+        self.subprocess.check_output.assert_called_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zone', 'modify',
+            '--rgw-zone=test_zone',
+            '--rgw-realm=test_realm',
+            '--rgw-zonegroup=test_zonegroup',
+            '--endpoints=http://localhost:80',
+            '--master', '--default', '--read-only=0',
+        ], stderr=mock.ANY)
+
+    @mock.patch.object(multisite, 'rename_zonegroup')
+    def test_rename_multisite_config_zone_fail(self, mock_rename_zonegroup):
+        mock_rename_zonegroup.return_value = True
+        self.assertEqual(
+            multisite.rename_multisite_config(
+                ['default'], 'test_zonegroup',
+                ['default'], 'test_zone'
+            ),
+            None
+        )
+
+        self.subprocess.call.assert_called_with([
+            'radosgw-admin', '--id=rgw.testhost',
+            'zone', 'rename', '--rgw-zone=default',
+            '--zone-new-name=test_zone',
+            '--rgw-zonegroup=test_zonegroup',
+        ])
+
+    @mock.patch.object(multisite, 'list_zonegroups')
+    @mock.patch.object(multisite, 'get_local_zone')
+    @mock.patch.object(multisite, 'list_buckets')
+    def test_check_zone_has_buckets(self, mock_list_zonegroups,
+                                    mock_get_local_zone,
+                                    mock_list_buckets):
+        mock_list_zonegroups.return_value = ['test_zonegroup']
+        mock_get_local_zone.return_value = 'test_zone', 'test_zonegroup'
+        mock_list_buckets.return_value = ['test_bucket_1', 'test_bucket_2']
+        self.assertEqual(
+            multisite.check_cluster_has_buckets(),
+            True
+        )