From 80b49853482a2a6f4993e7cdb3347d26d0d58260 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 17 Dec 2012 10:31:03 +0000 Subject: [PATCH] Resync with ceph charm, updates for raring --- README => README.md | 0 config.yaml | 17 +++++++ hooks/ceph.py | 108 ++++++++++++++++++++++++++++++++++++++++---- hooks/hooks.py | 44 ++++++++++++------ hooks/utils.py | 18 ++++++-- templates/ceph.conf | 6 +++ 6 files changed, 166 insertions(+), 27 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md diff --git a/config.yaml b/config.yaml index e8f9ad7f..497eb8f4 100644 --- a/config.yaml +++ b/config.yaml @@ -7,6 +7,23 @@ options: . These devices are the range of devices that will be checked for and used across all service units. + osd-journal: + type: string + description: | + The device to use as a shared journal drive for all OSD's. By default + no journal device will be used. + . + Only supported with ceph >= 0.55. + osd-format: + type: string + default: xfs + description: | + Format of filesystem to use for OSD devices; supported formats include: + . + xfs (Default >= 0.55) + ext4 (Only option < 0.55) + . + Only supported with ceph >= 0.55. osd-reformat: type: string description: | diff --git a/hooks/ceph.py b/hooks/ceph.py index 3e466c50..90a22a2d 100644 --- a/hooks/ceph.py +++ b/hooks/ceph.py @@ -12,8 +12,11 @@ import subprocess import time import utils import os +import apt_pkg as apt -QUORUM = ['leader', 'peon'] +LEADER = 'leader' +PEON = 'peon' +QUORUM = [LEADER, PEON] def is_quorum(): @@ -40,6 +43,30 @@ def is_quorum(): return False +def is_leader(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname()) + cmd = [ + "ceph", + "--admin-daemon", + asok, + "mon_status" + ] + if os.path.exists(asok): + try: + result = json.loads(subprocess.check_output(cmd)) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] == LEADER: + return True + else: + return False + else: + return False + + def wait_for_quorum(): while not is_quorum(): time.sleep(3) @@ -81,6 +108,7 @@ def rescan_osd_devices(): subprocess.call(cmd) + _bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring" @@ -88,6 +116,11 @@ def is_bootstrapped(): return os.path.exists(_bootstrap_keyring) +def wait_for_bootstrap(): + while (not is_bootstrapped()): + time.sleep(3) + + def import_osd_bootstrap_key(key): if not os.path.exists(_bootstrap_keyring): cmd = [ @@ -100,15 +133,53 @@ def import_osd_bootstrap_key(key): subprocess.check_call(cmd) # OSD caps taken from ceph-create-keys -_osd_bootstrap_caps = [ - 'allow command osd create ...', - 'allow command osd crush set ...', - r'allow command auth add * osd allow\ * mon allow\ rwx', - 'allow command mon getmap' - ] +_osd_bootstrap_caps = { + 'mon': [ + 'allow command osd create ...', + 'allow command osd crush set ...', + r'allow command auth add * osd allow\ * mon allow\ rwx', + 'allow command mon getmap' + ] + } def get_osd_bootstrap_key(): + return get_named_key('bootstrap-osd', _osd_bootstrap_caps) + + +_radosgw_keyring = "/etc/ceph/keyring.rados.gateway" + + +def import_radosgw_key(key): + if not os.path.exists(_radosgw_keyring): + cmd = [ + 'ceph-authtool', + _radosgw_keyring, + '--create-keyring', + '--name=client.radosgw.gateway', + '--add-key={}'.format(key) + ] + subprocess.check_call(cmd) + +# OSD caps taken from ceph-create-keys +_radosgw_caps = { + 'mon': ['allow r'], + 'osd': ['allow rwx'] + } + + +def get_radosgw_key(): + return get_named_key('radosgw.gateway', _radosgw_caps) + + +_default_caps = { + 'mon': ['allow r'], + 'osd': ['allow rwx'] + } + + +def get_named_key(name, caps=None): + caps = caps or _default_caps cmd = [ 'ceph', '--name', 'mon.', @@ -116,9 +187,14 @@ def get_osd_bootstrap_key(): '/var/lib/ceph/mon/ceph-{}/keyring'.format( utils.get_unit_hostname() ), - 'auth', 'get-or-create', 'client.bootstrap-osd', - 'mon', '; '.join(_osd_bootstrap_caps) + 'auth', 'get-or-create', 'client.{}'.format(name), ] + # Add capabilities + for subsystem, subcaps in caps.iteritems(): + cmd.extend([ + subsystem, + '; '.join(subcaps), + ]) output = subprocess.check_output(cmd).strip() # IGNORE:E1103 # get-or-create appears to have different output depending # on whether its 'get' or 'create' @@ -132,3 +208,17 @@ def get_osd_bootstrap_key(): if 'key' in element: key = element.split(' = ')[1].strip() # IGNORE:E1103 return key + + +def get_ceph_version(): + apt.init() + cache = apt.Cache() + pkg = cache['ceph'] + if pkg.current_ver: + return apt.upstream_version(pkg.current_ver.ver_str) + else: + return None + + +def version_compare(a, b): + return apt.version_compare(a, b) diff --git a/hooks/hooks.py b/hooks/hooks.py index a8d9d899..91fb2703 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -18,8 +18,11 @@ import utils def install_upstart_scripts(): - for x in glob.glob('files/upstart/*.conf'): - shutil.copy(x, '/etc/init/') + # Only install upstart configurations for older versions + if ceph.version_compare(ceph.get_ceph_version(), + "0.55.1") < 0: + for x in glob.glob('files/upstart/*.conf'): + shutil.copy(x, '/etc/init/') def install(): @@ -37,7 +40,8 @@ def emit_cephconf(): cephcontext = { 'auth_supported': get_auth(), 'mon_hosts': ' '.join(mon_hosts), - 'fsid': get_fsid() + 'fsid': get_fsid(), + 'version': ceph.get_ceph_version() } with open('/etc/ceph/ceph.conf', 'w') as cephconf: @@ -48,7 +52,7 @@ def config_changed(): utils.juju_log('INFO', 'Begin config-changed hook.') e_mountpoint = utils.config_get('ephemeral-unmount') - if (e_mountpoint != "" and + if (e_mountpoint and filesystem_mounted(e_mountpoint)): subprocess.call(['umount', e_mountpoint]) @@ -89,13 +93,13 @@ def get_conf(name): for unit in utils.relation_list(relid): conf = utils.relation_get(name, unit, relid) - if conf != "": + if conf: return conf return None def reformat_osd(): - if utils.config_get('osd-reformat') != "": + if utils.config_get('osd-reformat'): return True else: return False @@ -119,7 +123,24 @@ def osdize(dev): 'Looks like {} is in use, skipping.'.format(dev)) return - subprocess.call(['ceph-disk-prepare', dev]) + cmd = ['ceph-disk-prepare'] + # Later versions of ceph support more options + if ceph.version_compare(ceph.get_ceph_version(), + "0.55") >= 0: + osd_format = utils.config_get('osd-format') + if osd_format: + cmd.append('--fs-type') + cmd.append(osd_format) + cmd.append(dev) + osd_journal = utils.config_get('osd-journal') + if (osd_journal and + os.path.exists(osd_journal)): + cmd.append(osd_journal) + else: + # Just provide the device - no other options + # for older versions of ceph + cmd.append(dev) + subprocess.call(cmd) def device_mounted(dev): @@ -136,7 +157,7 @@ def mon_relation(): bootstrap_key = utils.relation_get('osd_bootstrap_key') if (get_fsid() and get_auth() and - bootstrap_key != ""): + bootstrap_key): utils.juju_log('INFO', 'mon has provided conf- scanning disks') emit_cephconf() ceph.import_osd_bootstrap_key(bootstrap_key) @@ -159,18 +180,11 @@ def upgrade_charm(): utils.juju_log('INFO', 'End upgrade-charm hook.') -def start(): - # In case we're being redeployed to the same machines, try - # to make sure everything is running as soon as possible. - ceph.rescan_osd_devices() - - utils.do_hooks({ 'config-changed': config_changed, 'install': install, 'mon-relation-departed': mon_relation, 'mon-relation-changed': mon_relation, - 'start': start, 'upgrade-charm': upgrade_charm, }) diff --git a/hooks/utils.py b/hooks/utils.py index 5ae96bc0..65d18fc2 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -136,7 +136,11 @@ def relation_get(attribute, unit=None, rid=None): cmd.append(attribute) if unit: cmd.append(unit) - return subprocess.check_output(cmd).strip() # IGNORE:E1103 + value = str(subprocess.check_output(cmd)).strip() + if value == "": + return None + else: + return value def relation_set(**kwargs): @@ -159,7 +163,11 @@ def unit_get(attribute): 'unit-get', attribute ] - return subprocess.check_output(cmd).strip() # IGNORE:E1103 + value = str(subprocess.check_output(cmd)).strip() + if value == "": + return None + else: + return value def config_get(attribute): @@ -167,7 +175,11 @@ def config_get(attribute): 'config-get', attribute ] - return subprocess.check_output(cmd).strip() # IGNORE:E1103 + value = str(subprocess.check_output(cmd)).strip() + if value == "": + return None + else: + return value def get_unit_hostname(): diff --git a/templates/ceph.conf b/templates/ceph.conf index 2af77d0a..887ba82f 100644 --- a/templates/ceph.conf +++ b/templates/ceph.conf @@ -1,5 +1,11 @@ [global] +{% if version < "0.51" %} auth supported = {{ auth_supported }} +{% else %} + auth cluster required = {{ auth_supported }} + auth service required = {{ auth_supported }} + auth client required = {{ auth_supported }} +{% endif %} keyring = /etc/ceph/$cluster.$name.keyring mon host = {{ mon_hosts }} fsid = {{ fsid }}