Rebase on charm-helpers

This commit is contained in:
James Page 2014-01-24 16:02:57 +00:00
parent f40d60fa56
commit 167a953ca4
21 changed files with 1294 additions and 266 deletions

8
Makefile Normal file

@ -0,0 +1,8 @@
#!/usr/bin/make
lint:
@flake8 --exclude hooks/charmhelpers hooks
@charm proof
sync:
@charm-helper-sync -c charm-helpers-sync.yaml

9
charm-helpers-sync.yaml Normal file

@ -0,0 +1,9 @@
branch: lp:charm-helpers
destination: hooks/charmhelpers
include:
- core
- fetch
- contrib.storage.linux:
- utils
- payload.execd
- contrib.openstack.alternatives

@ -10,23 +10,24 @@
import json
import subprocess
import time
import utils
import os
import apt_pkg as apt
import socket.gethostname as get_unit_hostname
LEADER = 'leader'
PEON = 'peon'
QUORUM = [LEADER, PEON]
def is_quorum():
asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname())
asok = "/var/run/ceph/ceph-mon.{}.asok".format(get_unit_hostname())
cmd = [
"ceph",
"--admin-daemon",
asok,
"mon_status"
]
]
if os.path.exists(asok):
try:
result = json.loads(subprocess.check_output(cmd))
@ -44,13 +45,13 @@ def is_quorum():
def is_leader():
asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname())
asok = "/var/run/ceph/ceph-mon.{}.asok".format(get_unit_hostname())
cmd = [
"ceph",
"--admin-daemon",
asok,
"mon_status"
]
]
if os.path.exists(asok):
try:
result = json.loads(subprocess.check_output(cmd))
@ -73,14 +74,14 @@ def wait_for_quorum():
def add_bootstrap_hint(peer):
asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname())
asok = "/var/run/ceph/ceph-mon.{}.asok".format(get_unit_hostname())
cmd = [
"ceph",
"--admin-daemon",
asok,
"add_bootstrap_peer_hint",
peer
]
]
if os.path.exists(asok):
# Ignore any errors for this call
subprocess.call(cmd)
@ -89,7 +90,7 @@ DISK_FORMATS = [
'xfs',
'ext4',
'btrfs'
]
]
def is_osd_disk(dev):
@ -99,7 +100,7 @@ def is_osd_disk(dev):
for line in info:
if line.startswith(
'Partition GUID code: 4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D'
):
):
return True
except subprocess.CalledProcessError:
pass
@ -110,7 +111,7 @@ def rescan_osd_devices():
cmd = [
'udevadm', 'trigger',
'--subsystem-match=block', '--action=add'
]
]
subprocess.call(cmd)
@ -140,7 +141,7 @@ def import_osd_bootstrap_key(key):
'--create-keyring',
'--name=client.bootstrap-osd',
'--add-key={}'.format(key)
]
]
subprocess.check_call(cmd)
# OSD caps taken from ceph-create-keys
@ -148,10 +149,10 @@ _osd_bootstrap_caps = {
'mon': [
'allow command osd create ...',
'allow command osd crush set ...',
r'allow command auth add * osd allow\ * mon allow\ rwx',
r'allow command auth add * osd allow\ * mon allow\ rwx',
'allow command mon getmap'
]
}
]
}
def get_osd_bootstrap_key():
@ -169,14 +170,14 @@ def import_radosgw_key(key):
'--create-keyring',
'--name=client.radosgw.gateway',
'--add-key={}'.format(key)
]
]
subprocess.check_call(cmd)
# OSD caps taken from ceph-create-keys
_radosgw_caps = {
'mon': ['allow r'],
'osd': ['allow rwx']
}
}
def get_radosgw_key():
@ -186,7 +187,7 @@ def get_radosgw_key():
_default_caps = {
'mon': ['allow r'],
'osd': ['allow rwx']
}
}
def get_named_key(name, caps=None):
@ -196,16 +197,16 @@ def get_named_key(name, caps=None):
'--name', 'mon.',
'--keyring',
'/var/lib/ceph/mon/ceph-{}/keyring'.format(
utils.get_unit_hostname()
),
get_unit_hostname()
),
'auth', 'get-or-create', 'client.{}'.format(name),
]
]
# Add capabilities
for subsystem, subcaps in caps.iteritems():
cmd.extend([
subsystem,
'; '.join(subcaps),
])
])
output = subprocess.check_output(cmd).strip() # IGNORE:E1103
# get-or-create appears to have different output depending
# on whether its 'get' or 'create'

@ -0,0 +1,17 @@
''' Helper for managing alternatives for file conflict resolution '''
import subprocess
import shutil
import os
def install_alternative(name, target, source, priority=50):
''' Install alternative configuration '''
if (os.path.exists(target) and not os.path.islink(target)):
# Move existing file/directory away before installing
shutil.move(target, '{}.bak'.format(target))
cmd = [
'update-alternatives', '--force', '--install',
target, name, source, str(priority)
]
subprocess.check_call(cmd)

@ -0,0 +1,25 @@
from os import stat
from stat import S_ISBLK
from subprocess import (
check_call
)
def is_block_device(path):
'''
Confirm device at path is a valid block device node.
:returns: boolean: True if path is a block device, False if not.
'''
return S_ISBLK(stat(path).st_mode)
def zap_disk(block_device):
'''
Clear a block device of partition table. Relies on sgdisk, which is
installed as pat of the 'gdisk' package in Ubuntu.
:param block_device: str: Full path of block device to clean.
'''
check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])

@ -0,0 +1,395 @@
"Interactions with the Juju environment"
# Copyright 2013 Canonical Ltd.
#
# Authors:
# Charm Helpers Developers <juju@lists.ubuntu.com>
import os
import json
import yaml
import subprocess
import UserDict
from subprocess import CalledProcessError
CRITICAL = "CRITICAL"
ERROR = "ERROR"
WARNING = "WARNING"
INFO = "INFO"
DEBUG = "DEBUG"
MARKER = object()
cache = {}
def cached(func):
"""Cache return values for multiple executions of func + args
For example:
@cached
def unit_get(attribute):
pass
unit_get('test')
will cache the result of unit_get + 'test' for future calls.
"""
def wrapper(*args, **kwargs):
global cache
key = str((func, args, kwargs))
try:
return cache[key]
except KeyError:
res = func(*args, **kwargs)
cache[key] = res
return res
return wrapper
def flush(key):
"""Flushes any entries from function cache where the
key is found in the function+args """
flush_list = []
for item in cache:
if key in item:
flush_list.append(item)
for item in flush_list:
del cache[item]
def log(message, level=None):
"""Write a message to the juju log"""
command = ['juju-log']
if level:
command += ['-l', level]
command += [message]
subprocess.call(command)
class Serializable(UserDict.IterableUserDict):
"""Wrapper, an object that can be serialized to yaml or json"""
def __init__(self, obj):
# wrap the object
UserDict.IterableUserDict.__init__(self)
self.data = obj
def __getattr__(self, attr):
# See if this object has attribute.
if attr in ("json", "yaml", "data"):
return self.__dict__[attr]
# Check for attribute in wrapped object.
got = getattr(self.data, attr, MARKER)
if got is not MARKER:
return got
# Proxy to the wrapped object via dict interface.
try:
return self.data[attr]
except KeyError:
raise AttributeError(attr)
def __getstate__(self):
# Pickle as a standard dictionary.
return self.data
def __setstate__(self, state):
# Unpickle into our wrapper.
self.data = state
def json(self):
"""Serialize the object to json"""
return json.dumps(self.data)
def yaml(self):
"""Serialize the object to yaml"""
return yaml.dump(self.data)
def execution_environment():
"""A convenient bundling of the current execution context"""
context = {}
context['conf'] = config()
if relation_id():
context['reltype'] = relation_type()
context['relid'] = relation_id()
context['rel'] = relation_get()
context['unit'] = local_unit()
context['rels'] = relations()
context['env'] = os.environ
return context
def in_relation_hook():
"""Determine whether we're running in a relation hook"""
return 'JUJU_RELATION' in os.environ
def relation_type():
"""The scope for the current relation hook"""
return os.environ.get('JUJU_RELATION', None)
def relation_id():
"""The relation ID for the current relation hook"""
return os.environ.get('JUJU_RELATION_ID', None)
def local_unit():
"""Local unit ID"""
return os.environ['JUJU_UNIT_NAME']
def remote_unit():
"""The remote unit for the current relation hook"""
return os.environ['JUJU_REMOTE_UNIT']
def service_name():
"""The name service group this unit belongs to"""
return local_unit().split('/')[0]
@cached
def config(scope=None):
"""Juju charm configuration"""
config_cmd_line = ['config-get']
if scope is not None:
config_cmd_line.append(scope)
config_cmd_line.append('--format=json')
try:
return json.loads(subprocess.check_output(config_cmd_line))
except ValueError:
return None
@cached
def relation_get(attribute=None, unit=None, rid=None):
"""Get relation information"""
_args = ['relation-get', '--format=json']
if rid:
_args.append('-r')
_args.append(rid)
_args.append(attribute or '-')
if unit:
_args.append(unit)
try:
return json.loads(subprocess.check_output(_args))
except ValueError:
return None
except CalledProcessError, e:
if e.returncode == 2:
return None
raise
def relation_set(relation_id=None, relation_settings={}, **kwargs):
"""Set relation information for the current unit"""
relation_cmd_line = ['relation-set']
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
for k, v in (relation_settings.items() + kwargs.items()):
if v is None:
relation_cmd_line.append('{}='.format(k))
else:
relation_cmd_line.append('{}={}'.format(k, v))
subprocess.check_call(relation_cmd_line)
# Flush cache of any relation-gets for local unit
flush(local_unit())
@cached
def relation_ids(reltype=None):
"""A list of relation_ids"""
reltype = reltype or relation_type()
relid_cmd_line = ['relation-ids', '--format=json']
if reltype is not None:
relid_cmd_line.append(reltype)
return json.loads(subprocess.check_output(relid_cmd_line)) or []
return []
@cached
def related_units(relid=None):
"""A list of related units"""
relid = relid or relation_id()
units_cmd_line = ['relation-list', '--format=json']
if relid is not None:
units_cmd_line.extend(('-r', relid))
return json.loads(subprocess.check_output(units_cmd_line)) or []
@cached
def relation_for_unit(unit=None, rid=None):
"""Get the json represenation of a unit's relation"""
unit = unit or remote_unit()
relation = relation_get(unit=unit, rid=rid)
for key in relation:
if key.endswith('-list'):
relation[key] = relation[key].split()
relation['__unit__'] = unit
return relation
@cached
def relations_for_id(relid=None):
"""Get relations of a specific relation ID"""
relation_data = []
relid = relid or relation_ids()
for unit in related_units(relid):
unit_data = relation_for_unit(unit, relid)
unit_data['__relid__'] = relid
relation_data.append(unit_data)
return relation_data
@cached
def relations_of_type(reltype=None):
"""Get relations of a specific type"""
relation_data = []
reltype = reltype or relation_type()
for relid in relation_ids(reltype):
for relation in relations_for_id(relid):
relation['__relid__'] = relid
relation_data.append(relation)
return relation_data
@cached
def relation_types():
"""Get a list of relation types supported by this charm"""
charmdir = os.environ.get('CHARM_DIR', '')
mdf = open(os.path.join(charmdir, 'metadata.yaml'))
md = yaml.safe_load(mdf)
rel_types = []
for key in ('provides', 'requires', 'peers'):
section = md.get(key)
if section:
rel_types.extend(section.keys())
mdf.close()
return rel_types
@cached
def relations():
"""Get a nested dictionary of relation data for all related units"""
rels = {}
for reltype in relation_types():
relids = {}
for relid in relation_ids(reltype):
units = {local_unit(): relation_get(unit=local_unit(), rid=relid)}
for unit in related_units(relid):
reldata = relation_get(unit=unit, rid=relid)
units[unit] = reldata
relids[relid] = units
rels[reltype] = relids
return rels
@cached
def is_relation_made(relation, keys='private-address'):
'''
Determine whether a relation is established by checking for
presence of key(s). If a list of keys is provided, they
must all be present for the relation to be identified as made
'''
if isinstance(keys, str):
keys = [keys]
for r_id in relation_ids(relation):
for unit in related_units(r_id):
context = {}
for k in keys:
context[k] = relation_get(k, rid=r_id,
unit=unit)
if None not in context.values():
return True
return False
def open_port(port, protocol="TCP"):
"""Open a service network port"""
_args = ['open-port']
_args.append('{}/{}'.format(port, protocol))
subprocess.check_call(_args)
def close_port(port, protocol="TCP"):
"""Close a service network port"""
_args = ['close-port']
_args.append('{}/{}'.format(port, protocol))
subprocess.check_call(_args)
@cached
def unit_get(attribute):
"""Get the unit ID for the remote unit"""
_args = ['unit-get', '--format=json', attribute]
try:
return json.loads(subprocess.check_output(_args))
except ValueError:
return None
def unit_private_ip():
"""Get this unit's private IP address"""
return unit_get('private-address')
class UnregisteredHookError(Exception):
"""Raised when an undefined hook is called"""
pass
class Hooks(object):
"""A convenient handler for hook functions.
Example:
hooks = Hooks()
# register a hook, taking its name from the function name
@hooks.hook()
def install():
...
# register a hook, providing a custom hook name
@hooks.hook("config-changed")
def config_changed():
...
if __name__ == "__main__":
# execute a hook based on the name the program is called by
hooks.execute(sys.argv)
"""
def __init__(self):
super(Hooks, self).__init__()
self._hooks = {}
def register(self, name, function):
"""Register a hook"""
self._hooks[name] = function
def execute(self, args):
"""Execute a registered hook based on args[0]"""
hook_name = os.path.basename(args[0])
if hook_name in self._hooks:
self._hooks[hook_name]()
else:
raise UnregisteredHookError(hook_name)
def hook(self, *hook_names):
"""Decorator, registering them as hooks"""
def wrapper(decorated):
for hook_name in hook_names:
self.register(hook_name, decorated)
else:
self.register(decorated.__name__, decorated)
if '_' in decorated.__name__:
self.register(
decorated.__name__.replace('_', '-'), decorated)
return decorated
return wrapper
def charm_dir():
"""Return the root directory of the current charm"""
return os.environ.get('CHARM_DIR')

@ -0,0 +1,291 @@
"""Tools for working with the host system"""
# Copyright 2012 Canonical Ltd.
#
# Authors:
# Nick Moffitt <nick.moffitt@canonical.com>
# Matthew Wedgwood <matthew.wedgwood@canonical.com>
import os
import pwd
import grp
import random
import string
import subprocess
import hashlib
from collections import OrderedDict
from hookenv import log
def service_start(service_name):
"""Start a system service"""
return service('start', service_name)
def service_stop(service_name):
"""Stop a system service"""
return service('stop', service_name)
def service_restart(service_name):
"""Restart a system service"""
return service('restart', service_name)
def service_reload(service_name, restart_on_failure=False):
"""Reload a system service, optionally falling back to restart if reload fails"""
service_result = service('reload', service_name)
if not service_result and restart_on_failure:
service_result = service('restart', service_name)
return service_result
def service(action, service_name):
"""Control a system service"""
cmd = ['service', service_name, action]
return subprocess.call(cmd) == 0
def service_running(service):
"""Determine whether a system service is running"""
try:
output = subprocess.check_output(['service', service, 'status'])
except subprocess.CalledProcessError:
return False
else:
if ("start/running" in output or "is running" in output):
return True
else:
return False
def adduser(username, password=None, shell='/bin/bash', system_user=False):
"""Add a user to the system"""
try:
user_info = pwd.getpwnam(username)
log('user {0} already exists!'.format(username))
except KeyError:
log('creating user {0}'.format(username))
cmd = ['useradd']
if system_user or password is None:
cmd.append('--system')
else:
cmd.extend([
'--create-home',
'--shell', shell,
'--password', password,
])
cmd.append(username)
subprocess.check_call(cmd)
user_info = pwd.getpwnam(username)
return user_info
def add_user_to_group(username, group):
"""Add a user to a group"""
cmd = [
'gpasswd', '-a',
username,
group
]
log("Adding user {} to group {}".format(username, group))
subprocess.check_call(cmd)
def rsync(from_path, to_path, flags='-r', options=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
cmd = ['/usr/bin/rsync', flags]
cmd.extend(options)
cmd.append(from_path)
cmd.append(to_path)
log(" ".join(cmd))
return subprocess.check_output(cmd).strip()
def symlink(source, destination):
"""Create a symbolic link"""
log("Symlinking {} as {}".format(source, destination))
cmd = [
'ln',
'-sf',
source,
destination,
]
subprocess.check_call(cmd)
def mkdir(path, owner='root', group='root', perms=0555, force=False):
"""Create a directory"""
log("Making dir {} {}:{} {:o}".format(path, owner, group,
perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
realpath = os.path.abspath(path)
if os.path.exists(realpath):
if force and not os.path.isdir(realpath):
log("Removing non-directory file {} prior to mkdir()".format(path))
os.unlink(realpath)
else:
os.makedirs(realpath, perms)
os.chown(realpath, uid, gid)
def write_file(path, content, owner='root', group='root', perms=0444):
"""Create or overwrite a file with the contents of a string"""
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
with open(path, 'w') as target:
os.fchown(target.fileno(), uid, gid)
os.fchmod(target.fileno(), perms)
target.write(content)
def mount(device, mountpoint, options=None, persist=False):
"""Mount a filesystem at a particular mountpoint"""
cmd_args = ['mount']
if options is not None:
cmd_args.extend(['-o', options])
cmd_args.extend([device, mountpoint])
try:
subprocess.check_output(cmd_args)
except subprocess.CalledProcessError, e:
log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
return False
if persist:
# TODO: update fstab
pass
return True
def umount(mountpoint, persist=False):
"""Unmount a filesystem"""
cmd_args = ['umount', mountpoint]
try:
subprocess.check_output(cmd_args)
except subprocess.CalledProcessError, e:
log('Error unmounting {}\n{}'.format(mountpoint, e.output))
return False
if persist:
# TODO: update fstab
pass
return True
def mounts():
"""Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
with open('/proc/mounts') as f:
# [['/mount/point','/dev/path'],[...]]
system_mounts = [m[1::-1] for m in [l.strip().split()
for l in f.readlines()]]
return system_mounts
def file_hash(path):
"""Generate a md5 hash of the contents of 'path' or None if not found """
if os.path.exists(path):
h = hashlib.md5()
with open(path, 'r') as source:
h.update(source.read()) # IGNORE:E1101 - it does have update
return h.hexdigest()
else:
return None
def restart_on_change(restart_map):
"""Restart services based on configuration files changing
This function is used a decorator, for example
@restart_on_change({
'/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
})
def ceph_client_changed():
...
In this example, the cinder-api and cinder-volume services
would be restarted if /etc/ceph/ceph.conf is changed by the
ceph_client_changed function.
"""
def wrap(f):
def wrapped_f(*args):
checksums = {}
for path in restart_map:
checksums[path] = file_hash(path)
f(*args)
restarts = []
for path in restart_map:
if checksums[path] != file_hash(path):
restarts += restart_map[path]
for service_name in list(OrderedDict.fromkeys(restarts)):
service('restart', service_name)
return wrapped_f
return wrap
def lsb_release():
"""Return /etc/lsb-release in a dict"""
d = {}
with open('/etc/lsb-release', 'r') as lsb:
for l in lsb:
k, v = l.split('=')
d[k.strip()] = v.strip()
return d
def pwgen(length=None):
"""Generate a random pasword."""
if length is None:
length = random.choice(range(35, 45))
alphanumeric_chars = [
l for l in (string.letters + string.digits)
if l not in 'l0QD1vAEIOUaeiou']
random_chars = [
random.choice(alphanumeric_chars) for _ in range(length)]
return(''.join(random_chars))
def list_nics(nic_type):
'''Return a list of nics of given type(s)'''
if isinstance(nic_type, basestring):
int_types = [nic_type]
else:
int_types = nic_type
interfaces = []
for int_type in int_types:
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
ip_output = subprocess.check_output(cmd).split('\n')
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
interfaces.append(line.split()[1].replace(":", ""))
return interfaces
def set_nic_mtu(nic, mtu):
'''Set MTU on a network interface'''
cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
subprocess.check_call(cmd)
def get_nic_mtu(nic):
cmd = ['ip', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).split('\n')
mtu = ""
for line in ip_output:
words = line.split()
if 'mtu' in words:
mtu = words[words.index("mtu") + 1]
return mtu
def get_nic_hwaddr(nic):
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd)
hwaddr = ""
words = ip_output.split()
if 'link/ether' in words:
hwaddr = words[words.index('link/ether') + 1]
return hwaddr

@ -0,0 +1,279 @@
import importlib
from yaml import safe_load
from charmhelpers.core.host import (
lsb_release
)
from urlparse import (
urlparse,
urlunparse,
)
import subprocess
from charmhelpers.core.hookenv import (
config,
log,
)
import apt_pkg
import os
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
"""
PROPOSED_POCKET = """# Proposed
deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
"""
CLOUD_ARCHIVE_POCKETS = {
# Folsom
'folsom': 'precise-updates/folsom',
'precise-folsom': 'precise-updates/folsom',
'precise-folsom/updates': 'precise-updates/folsom',
'precise-updates/folsom': 'precise-updates/folsom',
'folsom/proposed': 'precise-proposed/folsom',
'precise-folsom/proposed': 'precise-proposed/folsom',
'precise-proposed/folsom': 'precise-proposed/folsom',
# Grizzly
'grizzly': 'precise-updates/grizzly',
'precise-grizzly': 'precise-updates/grizzly',
'precise-grizzly/updates': 'precise-updates/grizzly',
'precise-updates/grizzly': 'precise-updates/grizzly',
'grizzly/proposed': 'precise-proposed/grizzly',
'precise-grizzly/proposed': 'precise-proposed/grizzly',
'precise-proposed/grizzly': 'precise-proposed/grizzly',
# Havana
'havana': 'precise-updates/havana',
'precise-havana': 'precise-updates/havana',
'precise-havana/updates': 'precise-updates/havana',
'precise-updates/havana': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'precise-havana/proposed': 'precise-proposed/havana',
'precise-proposed/havana': 'precise-proposed/havana',
# Icehouse
'icehouse': 'precise-updates/icehouse',
'precise-icehouse': 'precise-updates/icehouse',
'precise-icehouse/updates': 'precise-updates/icehouse',
'precise-updates/icehouse': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
'precise-icehouse/proposed': 'precise-proposed/icehouse',
'precise-proposed/icehouse': 'precise-proposed/icehouse',
}
def filter_installed_packages(packages):
"""Returns a list of packages that require installation"""
apt_pkg.init()
cache = apt_pkg.Cache()
_pkgs = []
for package in packages:
try:
p = cache[package]
p.current_ver or _pkgs.append(package)
except KeyError:
log('Package {} has no installation candidate.'.format(package),
level='WARNING')
_pkgs.append(package)
return _pkgs
def apt_install(packages, options=None, fatal=False):
"""Install one or more packages"""
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
cmd = ['apt-get', '--assume-yes']
cmd.extend(options)
cmd.append('install')
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Installing {} with options: {}".format(packages,
options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd, env=env)
def apt_update(fatal=False):
"""Update local apt cache"""
cmd = ['apt-get', 'update']
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def apt_purge(packages, fatal=False):
"""Purge one or more packages"""
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Purging {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def apt_hold(packages, fatal=False):
"""Hold one or more packages"""
cmd = ['apt-mark', 'hold']
if isinstance(packages, basestring):
cmd.append(packages)
else:
cmd.extend(packages)
log("Holding {}".format(packages))
if fatal:
subprocess.check_call(cmd)
else:
subprocess.call(cmd)
def add_source(source, key=None):
if (source.startswith('ppa:') or
source.startswith('http:') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
subprocess.check_call(['add-apt-repository', '--yes', source])
elif source.startswith('cloud:'):
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
pocket = source.split(':')[-1]
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
elif source == 'proposed':
release = lsb_release()['DISTRIB_CODENAME']
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(PROPOSED_POCKET.format(release))
if key:
subprocess.check_call(['apt-key', 'import', key])
class SourceConfigError(Exception):
pass
def configure_sources(update=False,
sources_var='install_sources',
keys_var='install_keys'):
"""
Configure multiple sources from charm configuration
Example config:
install_sources:
- "ppa:foo"
- "http://example.com/repo precise main"
install_keys:
- null
- "a1b2c3d4"
Note that 'null' (a.k.a. None) should not be quoted.
"""
sources = safe_load(config(sources_var))
keys = config(keys_var)
if keys is not None:
keys = safe_load(keys)
if isinstance(sources, basestring) and (
keys is None or isinstance(keys, basestring)):
add_source(sources, keys)
else:
if not len(sources) == len(keys):
msg = 'Install sources and keys lists are different lengths'
raise SourceConfigError(msg)
for src_num in range(len(sources)):
add_source(sources[src_num], keys[src_num])
if update:
apt_update(fatal=True)
# The order of this list is very important. Handlers should be listed in from
# least- to most-specific URL matching.
FETCH_HANDLERS = (
'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
)
class UnhandledSource(Exception):
pass
def install_remote(source):
"""
Install a file tree from a remote source
The specified source should be a url of the form:
scheme://[host]/path[#[option=value][&...]]
Schemes supported are based on this modules submodules
Options supported are submodule-specific"""
# We ONLY check for True here because can_handle may return a string
# explaining why it can't handle a given source.
handlers = [h for h in plugins() if h.can_handle(source) is True]
installed_to = None
for handler in handlers:
try:
installed_to = handler.install(source)
except UnhandledSource:
pass
if not installed_to:
raise UnhandledSource("No handler found for source {}".format(source))
return installed_to
def install_from_config(config_var_name):
charm_config = config()
source = charm_config[config_var_name]
return install_remote(source)
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
return "Wrong source type"
def install(self, source):
"""Try to download and unpack the source. Return the path to the
unpacked files or raise UnhandledSource."""
raise UnhandledSource("Wrong source type {}".format(source))
def parse_url(self, url):
return urlparse(url)
def base_url(self, url):
"""Return url without querystring or fragment"""
parts = list(self.parse_url(url))
parts[4:] = ['' for i in parts[4:]]
return urlunparse(parts)
def plugins(fetch_handlers=None):
if not fetch_handlers:
fetch_handlers = FETCH_HANDLERS
plugin_list = []
for handler_name in fetch_handlers:
package, classname = handler_name.rsplit('.', 1)
try:
handler_class = getattr(
importlib.import_module(package),
classname)
plugin_list.append(handler_class())
except (ImportError, AttributeError):
# Skip missing plugins so that they can be ommitted from
# installation if desired
log("FetchHandler {} not found, skipping plugin".format(
handler_name))
return plugin_list

@ -0,0 +1,48 @@
import os
import urllib2
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource
)
from charmhelpers.payload.archive import (
get_archive_handler,
extract,
)
from charmhelpers.core.host import mkdir
class ArchiveUrlFetchHandler(BaseFetchHandler):
"""Handler for archives via generic URLs"""
def can_handle(self, source):
url_parts = self.parse_url(source)
if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
return "Wrong source type"
if get_archive_handler(self.base_url(source)):
return True
return False
def download(self, source, dest):
# propogate all exceptions
# URLError, OSError, etc
response = urllib2.urlopen(source)
try:
with open(dest, 'w') as dest_file:
dest_file.write(response.read())
except Exception as e:
if os.path.isfile(dest):
os.unlink(dest)
raise e
def install(self, source):
url_parts = self.parse_url(source)
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
try:
self.download(source, dld_file)
except urllib2.URLError as e:
raise UnhandledSource(e.reason)
except OSError as e:
raise UnhandledSource(e.strerror)
return extract(dld_file)

@ -0,0 +1,49 @@
import os
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource
)
from charmhelpers.core.host import mkdir
try:
from bzrlib.branch import Branch
except ImportError:
from charmhelpers.fetch import apt_install
apt_install("python-bzrlib")
from bzrlib.branch import Branch
class BzrUrlFetchHandler(BaseFetchHandler):
"""Handler for bazaar branches via generic and lp URLs"""
def can_handle(self, source):
url_parts = self.parse_url(source)
if url_parts.scheme not in ('bzr+ssh', 'lp'):
return False
else:
return True
def branch(self, source, dest):
url_parts = self.parse_url(source)
# If we use lp:branchname scheme we need to load plugins
if not self.can_handle(source):
raise UnhandledSource("Cannot handle {}".format(source))
if url_parts.scheme == "lp":
from bzrlib.plugin import load_plugins
load_plugins()
try:
remote_branch = Branch.open(source)
remote_branch.bzrdir.sprout(dest).open_branch()
except Exception as e:
raise e
def install(self, source):
url_parts = self.parse_url(source)
branch_name = url_parts.path.strip("/").split("/")[-1]
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
if not os.path.exists(dest_dir):
mkdir(dest_dir, perms=0755)
try:
self.branch(source, dest_dir)
except OSError as e:
raise UnhandledSource(e.strerror)
return dest_dir

@ -0,0 +1 @@
"Tools for working with files injected into a charm just before deployment."

@ -0,0 +1,50 @@
#!/usr/bin/env python
import os
import sys
import subprocess
from charmhelpers.core import hookenv
def default_execd_dir():
return os.path.join(os.environ['CHARM_DIR'], 'exec.d')
def execd_module_paths(execd_dir=None):
"""Generate a list of full paths to modules within execd_dir."""
if not execd_dir:
execd_dir = default_execd_dir()
if not os.path.exists(execd_dir):
return
for subpath in os.listdir(execd_dir):
module = os.path.join(execd_dir, subpath)
if os.path.isdir(module):
yield module
def execd_submodule_paths(command, execd_dir=None):
"""Generate a list of full paths to the specified command within exec_dir.
"""
for module_path in execd_module_paths(execd_dir):
path = os.path.join(module_path, command)
if os.access(path, os.X_OK) and os.path.isfile(path):
yield path
def execd_run(command, execd_dir=None, die_on_error=False, stderr=None):
"""Run command for each module within execd_dir which defines it."""
for submodule_path in execd_submodule_paths(command, execd_dir):
try:
subprocess.check_call(submodule_path, shell=True, stderr=stderr)
except subprocess.CalledProcessError as e:
hookenv.log("Error ({}) running {}. Output: {}".format(
e.returncode, e.cmd, e.output))
if die_on_error:
sys.exit(e.returncode)
def execd_preinstall(execd_dir=None):
"""Run charm-pre-install for each module within execd_dir."""
execd_run('charm-pre-install', execd_dir=execd_dir)

@ -14,7 +14,32 @@ import glob
import os
import ceph
import utils
from charmhelpers.core.hookenv import (
relation_get,
relation_ids,
related_units,
config,
unit_get,
open_port,
relation_set,
log,
Hooks, UnregisteredHookError,
)
from charmhelpers.fetch import (
apt_update,
apt_install,
add_source,
)
from utils import (
render_template,
get_host_ip,
enable_pocket
)
from charmhelpers.payload.execd import execd_preinstall
import socket.gethostname as get_unit_hostname
hooks = Hooks()
def install_www_scripts():
@ -22,19 +47,20 @@ def install_www_scripts():
shutil.copy(x, '/var/www/')
NSS_DIR='/var/lib/ceph/nss'
NSS_DIR = '/var/lib/ceph/nss'
@hooks.hook('install')
def install():
utils.juju_log('INFO', 'Begin install hook.')
utils.enable_pocket('multiverse')
utils.configure_source()
utils.install('radosgw',
'libapache2-mod-fastcgi',
'apache2',
'ntp')
execd_preinstall()
enable_pocket('multiverse')
add_source(config('source'), config('key'))
apt_update(fatal=True)
apt_install(['radosgw',
'libapache2-mod-fastcgi',
'apache2',
'ntp'], fatal=True)
os.makedirs(NSS_DIR)
utils.juju_log('INFO', 'End install hook.')
def emit_cephconf():
@ -45,71 +71,67 @@ def emit_cephconf():
cephcontext = {
'auth_supported': get_auth() or 'none',
'mon_hosts': ' '.join(get_mon_hosts()),
'hostname': utils.get_unit_hostname(),
'hostname': get_unit_hostname(),
'version': ceph.get_ceph_version('radosgw')
}
# Check to ensure that correct version of ceph is
}
# Check to ensure that correct version of ceph is
# in use
if ceph.get_ceph_version('radosgw') >= "0.55":
if ceph.get_ceph_version('radosgw') >= "0.55":
# Add keystone configuration if found
ks_conf = get_keystone_conf()
if ks_conf:
cephcontext.update(ks_conf)
with open('/etc/ceph/ceph.conf', 'w') as cephconf:
cephconf.write(utils.render_template('ceph.conf', cephcontext))
cephconf.write(render_template('ceph.conf', cephcontext))
def emit_apacheconf():
apachecontext = {
"hostname": utils.unit_get('private-address')
}
"hostname": unit_get('private-address')
}
with open('/etc/apache2/sites-available/rgw.conf', 'w') as apacheconf:
apacheconf.write(utils.render_template('rgw', apachecontext))
apacheconf.write(render_template('rgw', apachecontext))
def apache_sites():
utils.juju_log('INFO', 'Begin apache_sites.')
if os.path.exists('/etc/apache2/sites-available/000-default.conf'):
subprocess.check_call(['a2dissite', '000-default'])
else:
subprocess.check_call(['a2dissite', 'default'])
subprocess.check_call(['a2ensite', 'rgw'])
utils.juju_log('INFO', 'End apache_sites.')
def apache_modules():
utils.juju_log('INFO', 'Begin apache_sites.')
subprocess.check_call(['a2enmod', 'fastcgi'])
subprocess.check_call(['a2enmod', 'rewrite'])
utils.juju_log('INFO', 'End apache_sites.')
def apache_reload():
subprocess.call(['service', 'apache2', 'reload'])
@hooks.hook('upgrade-charm',
'config-changed')
def config_changed():
utils.juju_log('INFO', 'Begin config-changed hook.')
emit_cephconf()
emit_apacheconf()
install_www_scripts()
apache_sites()
apache_modules()
apache_reload()
utils.juju_log('INFO', 'End config-changed hook.')
def get_mon_hosts():
hosts = []
for relid in utils.relation_ids('mon'):
for unit in utils.relation_list(relid):
for relid in relation_ids('mon'):
for unit in related_units(relid):
hosts.append(
'{}:6789'.format(utils.get_host_ip(
utils.relation_get('private-address',
unit, relid)))
)
'{}:6789'.format(get_host_ip(
relation_get('private-address',
unit, relid)))
)
hosts.sort()
return hosts
@ -120,100 +142,90 @@ def get_auth():
def get_conf(name):
for relid in utils.relation_ids('mon'):
for unit in utils.relation_list(relid):
conf = utils.relation_get(name,
unit, relid)
for relid in relation_ids('mon'):
for unit in related_units(relid):
conf = relation_get(name,
unit, relid)
if conf:
return conf
return None
def get_keystone_conf():
for relid in utils.relation_ids('identity-service'):
for unit in utils.relation_list(relid):
for relid in relation_ids('identity-service'):
for unit in related_units(relid):
ks_auth = {
'auth_type': 'keystone',
'auth_protocol': 'http',
'auth_host': utils.relation_get('auth_host', unit, relid),
'auth_port': utils.relation_get('auth_port', unit, relid),
'admin_token': utils.relation_get('admin_token', unit, relid),
'user_roles': utils.config_get('operator-roles'),
'cache_size': utils.config_get('cache-size'),
'revocation_check_interval': utils.config_get('revocation-check-interval')
'auth_host': relation_get('auth_host', unit, relid),
'auth_port': relation_get('auth_port', unit, relid),
'admin_token': relation_get('admin_token', unit, relid),
'user_roles': config('operator-roles'),
'cache_size': config('cache-size'),
'revocation_check_interval':
config('revocation-check-interval')
}
if None not in ks_auth.itervalues():
return ks_auth
return None
@hooks.hook('mon-relation-departed',
'mon-relation-changed')
def mon_relation():
utils.juju_log('INFO', 'Begin mon-relation hook.')
emit_cephconf()
key = utils.relation_get('radosgw_key')
key = relation_get('radosgw_key')
if key:
ceph.import_radosgw_key(key)
restart() # TODO figure out a better way todo this
utils.juju_log('INFO', 'End mon-relation hook.')
@hooks.hook('gateway-relation-joined')
def gateway_relation():
utils.juju_log('INFO', 'Begin gateway-relation hook.')
utils.relation_set(hostname=utils.unit_get('private-address'),
port=80)
utils.juju_log('INFO', 'Begin gateway-relation hook.')
def upgrade_charm():
utils.juju_log('INFO', 'Begin upgrade-charm hook.')
utils.juju_log('INFO', 'End upgrade-charm hook.')
relation_set(hostname=unit_get('private-address'),
port=80)
def start():
subprocess.call(['service', 'radosgw', 'start'])
utils.expose(port=80)
open_port(port=80)
def stop():
subprocess.call(['service', 'radosgw', 'stop'])
utils.expose(port=80)
open_port(port=80)
def restart():
subprocess.call(['service', 'radosgw', 'restart'])
utils.expose(port=80)
open_port(port=80)
@hooks.hook('identity-service-relation-joined',
'identity-service-relation-changed')
def identity_joined(relid=None):
if ceph.get_ceph_version('radosgw') < "0.55":
utils.juju_log('ERROR',
'Integration with keystone requires ceph >= 0.55')
log('Integration with keystone requires ceph >= 0.55')
sys.exit(1)
hostname = utils.unit_get('private-address')
hostname = unit_get('private-address')
admin_url = 'http://{}:80/swift'.format(hostname)
internal_url = public_url = '{}/v1'.format(admin_url)
utils.relation_set(service='swift',
region=utils.config_get('region'),
public_url=public_url, internal_url=internal_url,
admin_url=admin_url,
requested_roles=utils.config_get('operator-roles'),
rid=relid)
relation_set(service='swift',
region=config('region'),
public_url=public_url, internal_url=internal_url,
admin_url=admin_url,
requested_roles=config('operator-roles'),
rid=relid)
def identity_changed():
emit_cephconf()
restart()
restart()
utils.do_hooks({
'install': install,
'config-changed': config_changed,
'mon-relation-departed': mon_relation,
'mon-relation-changed': mon_relation,
'gateway-relation-joined': gateway_relation,
'upgrade-charm': config_changed, # same function ATM
'identity-service-relation-joined': identity_joined,
'identity-service-relation-changed': identity_changed
})
sys.exit(0)
if __name__ == '__main__':
try:
hooks.execute(sys.argv)
except UnregisteredHookError as e:
log('Unknown hook {} - skipping.'.format(e))

@ -7,97 +7,35 @@
# Paul Collins <paul.collins@canonical.com>
#
import os
import subprocess
import socket
import sys
import re
def do_hooks(hooks):
hook = os.path.basename(sys.argv[0])
try:
hook_func = hooks[hook]
except KeyError:
juju_log('INFO',
"This charm doesn't know how to handle '{}'.".format(hook))
else:
hook_func()
def install(*pkgs):
cmd = [
'apt-get',
'-y',
'install'
]
for pkg in pkgs:
cmd.append(pkg)
subprocess.check_call(cmd)
from charmhelpers.core.hookenv import unit_get
from charmhelpers.fetch import apt_install
TEMPLATES_DIR = 'templates'
try:
import jinja2
except ImportError:
install('python-jinja2')
apt_install('python-jinja2', fatal=True)
import jinja2
try:
import dns.resolver
except ImportError:
install('python-dnspython')
apt_install('python-dnspython', fatal=True)
import dns.resolver
def render_template(template_name, context, template_dir=TEMPLATES_DIR):
templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir)
)
loader=jinja2.FileSystemLoader(template_dir)
)
template = templates.get_template(template_name)
return template.render(context)
CLOUD_ARCHIVE = \
""" # Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
"""
def configure_source():
source = str(config_get('source'))
if not source:
return
if source.startswith('ppa:'):
cmd = [
'add-apt-repository',
source
]
subprocess.check_call(cmd)
if source.startswith('cloud:'):
install('ubuntu-cloud-keyring')
pocket = source.split(':')[1]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(pocket))
if source.startswith('http:'):
with open('/etc/apt/sources.list.d/quantum.list', 'w') as apt:
apt.write("deb " + source + "\n")
key = config_get('key')
if key:
cmd = [
'apt-key',
'adv', '--keyserver keyserver.ubuntu.com',
'--recv-keys', key
]
subprocess.check_call(cmd)
cmd = [
'apt-get',
'update'
]
subprocess.check_call(cmd)
def enable_pocket(pocket):
apt_sources = "/etc/apt/sources.list"
with open(apt_sources, "r") as sources:
@ -109,103 +47,6 @@ def enable_pocket(pocket):
else:
sources.write(line)
# Protocols
TCP = 'TCP'
UDP = 'UDP'
def expose(port, protocol='TCP'):
cmd = [
'open-port',
'{}/{}'.format(port, protocol)
]
subprocess.check_call(cmd)
def juju_log(severity, message):
cmd = [
'juju-log',
'--log-level', severity,
message
]
subprocess.check_call(cmd)
def relation_ids(relation):
cmd = [
'relation-ids',
relation
]
return subprocess.check_output(cmd).split() # IGNORE:E1103
def relation_list(rid):
cmd = [
'relation-list',
'-r', rid,
]
return subprocess.check_output(cmd).split() # IGNORE:E1103
def relation_get(attribute, unit=None, rid=None):
cmd = [
'relation-get',
]
if rid:
cmd.append('-r')
cmd.append(rid)
cmd.append(attribute)
if unit:
cmd.append(unit)
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def relation_set(**kwargs):
cmd = [
'relation-set'
]
args = []
for k, v in kwargs.items():
if k == 'rid' and v:
cmd.append('-r')
cmd.append(v)
elif k != 'rid':
args.append('{}={}'.format(k, v))
cmd += args
subprocess.check_call(cmd)
def unit_get(attribute):
cmd = [
'unit-get',
attribute
]
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def config_get(attribute):
cmd = [
'config-get',
attribute
]
value = str(subprocess.check_output(cmd)).strip()
if value == "":
return None
else:
return value
def get_unit_hostname():
return socket.gethostname()
def get_host_ip(hostname=unit_get('private-address')):
try:

@ -7,6 +7,8 @@ description: |
.
This charm provides the RADOS HTTP gateway supporting S3 and Swift protocols
for object storage.
categories:
- misc
requires:
mon:
interface: ceph-radosgw