diff --git a/Makefile b/Makefile index 0af1d0c2..eab0e2ea 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,7 @@ test: # coreycb note: The -v should only be temporary until Amulet sends # raise_status() messages to stderr: # https://bugs.launchpad.net/amulet/+bug/1320357 - @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ - 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse + @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700 bin/charm_helpers_sync.py: @mkdir -p bin diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..11d49a7c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,37 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, + self.utopic_juno, self.vivid_kilo) = range(10) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, + ('utopic', None): self.utopic_juno, + ('vivid', None): self.vivid_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c7c4cd4a..400eaf8e 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -16,11 +16,13 @@ import json import os +import re import time from base64 import b64decode from subprocess import check_call import six +import yaml from charmhelpers.fetch import ( apt_install, @@ -45,8 +47,11 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr, mkdir, write_file, ) @@ -63,16 +68,22 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, +) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + INTERNAL, ) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_ipv4_addr, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, is_address_in_network, + is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -104,9 +115,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -191,7 +234,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: @@ -277,12 +320,29 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): - interfaces = ['identity-service'] + + def __init__(self, service=None, service_user=None, rel_name='identity-service'): + self.service = service + self.service_user = service_user + self.rel_name = rel_name + self.interfaces = [self.rel_name] def __call__(self): - log('Generating template context for identity-service', level=DEBUG) + log('Generating template context for ' + self.rel_name, level=DEBUG) ctxt = {} - for rid in relation_ids('identity-service'): + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + + for rid in relation_ids(self.rel_name): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) serv_host = rdata.get('service_host') @@ -291,15 +351,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -398,6 +459,11 @@ class AMQPContext(OSContextGenerator): ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts)) + oslo_messaging_flags = conf.get('oslo-messaging-flags', None) + if oslo_messaging_flags: + ctxt['oslo_messaging_flags'] = config_flags_parser( + oslo_messaging_flags) + if not context_complete(ctxt): return {} @@ -677,7 +743,14 @@ class ApacheSSLContext(OSContextGenerator): 'endpoints': [], 'ext_ports': []} - for cn in self.canonical_names(): + cns = self.canonical_names() + if cns: + for cn in cns: + self.configure_cert(cn) + else: + # Expect cert/key provided in config (currently assumed that ca + # uses ip for cn) + cn = resolve_address(endpoint_type=INTERNAL) self.configure_cert(cn) addresses = self.get_network_addresses() @@ -740,6 +813,19 @@ class NeutronContext(OSContextGenerator): return ovs_ctxt + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + def nvp_ctxt(self): driver = neutron_plugin_attribute(self.plugin, 'driver', self.network_manager) @@ -823,6 +909,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.n1kv_ctxt()) elif self.plugin == 'Calico': ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -833,6 +921,48 @@ class NeutronContext(OSContextGenerator): return ctxt +class NeutronPortContext(OSContextGenerator): + NIC_PREFIXES = ['eth', 'bond'] + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(self.NIC_PREFIXES): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + resolved = [] + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in ports: + if re.match(mac_regex, entry): + # NIC is in known NICs and does NOT hace an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + resolved.append(hwaddr_to_nic[entry]) + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + resolved.append(entry) + + return resolved + + class OSConfigFlagContext(OSContextGenerator): """Provides support for user-defined config flags. @@ -1021,6 +1151,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt @@ -1052,3 +1184,145 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 902757fe..02c92e9c 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -16,6 +16,7 @@ # Various utilies for dealing with Neutron and the renaming from Quantum. +import six from subprocess import check_output from charmhelpers.core.hookenv import ( @@ -179,6 +180,19 @@ def neutron_plugins(): 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': @@ -237,3 +251,72 @@ def network_manager(): else: # ensure accurate naming for all releases post-H return 'neutron' + + +def parse_mappings(mappings): + parsed = {} + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + parsed[p[0].strip()] = p[2].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings, default_bridge='br-data'): + """Parse data port mappings. + + Mappings must be a space-delimited list of bridge:port mappings. + + Returns dict of the form {bridge:port}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {default_bridge: mappings.split(' ')[0]} + + bridges = _mappings.keys() + ports = _mappings.values() + if len(set(bridges)) != len(bridges): + raise Exception("It is not allowed to have more than one port " + "configured on the same bridge") + + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return _mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + return {} + + mappings = {} + for p, r in six.iteritems(_mappings): + mappings[p] = tuple(r.split(':')) + + return mappings diff --git a/hooks/charmhelpers/contrib/openstack/templates/git.upstart b/hooks/charmhelpers/contrib/openstack/templates/git.upstart new file mode 100644 index 00000000..4bed404b --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/git.upstart @@ -0,0 +1,17 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} + --config-file={{ config_file }} \ + {% endfor -%} + {% if log_file -%} + --log-file={{ log_file }} + {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken new file mode 100644 index 00000000..2a37edd5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -0,0 +1,9 @@ +{% if auth_host -%} +[keystone_authtoken] +identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo new file mode 100644 index 00000000..b444c9c9 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-zeromq b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq new file mode 100644 index 00000000..95f1a76c --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-zeromq @@ -0,0 +1,14 @@ +{% if zmq_host -%} +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) +rpc_backend = zmq +rpc_zmq_host = {{ zmq_host }} +{% if zmq_redis_address -%} +rpc_zmq_matchmaker = redis +matchmaker_heartbeat_freq = 15 +matchmaker_heartbeat_ttl = 30 +[matchmaker_redis] +host = {{ zmq_redis_address }} +{% else -%} +rpc_zmq_matchmaker = ring +{% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..f90a0289 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,12 +23,17 @@ from functools import wraps import subprocess import json import os -import socket import sys import six import yaml +from charmhelpers.contrib.network import ip + +from charmhelpers.core import ( + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -329,6 +334,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = config(option) + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -421,77 +441,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): @@ -535,82 +488,106 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git') is not None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" - global requirements_dir +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if file_name == "None": + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + http_proxy: http://squid.internal:3128 + https_proxy: https://squid.internal:3128 + + The directory, http_proxy, and https_proxy keys are optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - yaml_file = os.path.join(charm_dir(), file_name) + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, - whitelist=['requirements']) - if 'requirements' not in installed: - error_out('requirements git repository must be specified') + old_environ = dict(os.environ) - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + if 'http_proxy' in projects.keys(): + os.environ['http_proxy'] = projects['http_proxy'] + if 'https_proxy' in projects.keys(): + os.environ['https_proxy'] = projects['https_proxy'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) + requirements_dir = repo_dir + else: + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) + + os.environ = old_environ -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" - global requirements_dir - installed = [] +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) - return installed + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) -def _git_clone_and_install_single(repo, branch, update_requirements=False): - """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: repo_dir = dest_dir @@ -627,16 +604,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False): def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index cf552b39..86f805f1 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -20,11 +20,13 @@ # Authors: # Charm Helpers Developers +from __future__ import print_function import os import json import yaml import subprocess import sys +import errno from subprocess import CalledProcessError import six @@ -87,7 +89,18 @@ def log(message, level=None): if not isinstance(message, six.string_types): message = repr(message) command += [message] - subprocess.call(command) + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise class Serializable(UserDict): @@ -566,3 +579,29 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" return os.environ.get('CHARM_DIR') + + +@cached +def action_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +def action_set(values): + """Sets the values to be returned after the action finishes""" + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def action_fail(message): + """Sets the action status to failed and sets the error message. + + The results set by action_set are preserved.""" + subprocess.check_call(['action-fail', message]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b771c611..830822af 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -339,12 +339,16 @@ def lsb_release(): def pwgen(length=None): """Generate a random pasword.""" if length is None: + # A random length is ok to use a weak PRNG length = random.choice(range(35, 45)) alphanumeric_chars = [ l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() random_chars = [ - random.choice(alphanumeric_chars) for _ in range(length)] + random_generator.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..3eb5fb44 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + RelationContext.__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + RelationContext.__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py index efc4402e..a2a784aa 100644 --- a/hooks/charmhelpers/core/strutils.py +++ b/hooks/charmhelpers/core/strutils.py @@ -33,9 +33,9 @@ def bool_from_string(value): value = value.strip().lower() - if value in ['y', 'yes', 'true', 't']: + if value in ['y', 'yes', 'true', 't', 'on']: return True - elif value in ['n', 'no', 'false', 'f']: + elif value in ['n', 'no', 'false', 'f', 'off']: return False msg = "Unable to interpret string value '%s' as boolean" % (value) diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134a..406a35c5 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/14-basic-precise-icehouse b/tests/014-basic-precise-icehouse similarity index 100% rename from tests/14-basic-precise-icehouse rename to tests/014-basic-precise-icehouse diff --git a/tests/15-basic-trusty-icehouse b/tests/015-basic-trusty-icehouse similarity index 100% rename from tests/15-basic-trusty-icehouse rename to tests/015-basic-trusty-icehouse diff --git a/tests/016-basic-trusty-juno b/tests/016-basic-trusty-juno new file mode 100755 index 00000000..d5de9b14 --- /dev/null +++ b/tests/016-basic-trusty-juno @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic ceph-radosgw deployment on trusty-juno.""" + +from basic_deployment import CephRadosGwBasicDeployment + +if __name__ == '__main__': + deployment = CephRadosGwBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno') + deployment.run_tests() diff --git a/tests/017-basic-trusty-kilo b/tests/017-basic-trusty-kilo new file mode 100644 index 00000000..3335b188 --- /dev/null +++ b/tests/017-basic-trusty-kilo @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic ceph-radosgw deployment on trusty-kilo.""" + +from basic_deployment import CephRadosGwBasicDeployment + +if __name__ == '__main__': + deployment = CephRadosGwBasicDeployment(series='trusty', + openstack='cloud:trusty-kilo', + source='cloud:trusty-updates/kilo') + deployment.run_tests() diff --git a/tests/018-basic-utopic-juno b/tests/018-basic-utopic-juno new file mode 100755 index 00000000..4e4f5e59 --- /dev/null +++ b/tests/018-basic-utopic-juno @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic ceph-radosgw deployment on utopic-juno.""" + +from basic_deployment import CephRadosGwBasicDeployment + +if __name__ == '__main__': + deployment = CephRadosGwBasicDeployment(series='utopic') + deployment.run_tests() diff --git a/tests/019-basic-vivid-kilo b/tests/019-basic-vivid-kilo new file mode 100644 index 00000000..9238de85 --- /dev/null +++ b/tests/019-basic-vivid-kilo @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic ceph-radosgw deployment on vivid-kilo.""" + +from basic_deployment import CephRadosGwBasicDeployment + +if __name__ == '__main__': + deployment = CephRadosGwBasicDeployment(series='vivid') + deployment.run_tests() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 44c0e442..dd0d0ef1 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -11,7 +11,7 @@ from charmhelpers.contrib.openstack.amulet.utils import ( # noqa ) # Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(ERROR) +u = OpenStackAmuletUtils(DEBUG) class CephRadosGwBasicDeployment(OpenStackAmuletDeployment): diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 65219d33..5088b1d1 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -118,6 +118,9 @@ class AmuletUtils(object): longs, or can be a function that evaluate a variable and returns a bool. """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + for k, v in six.iteritems(expected): if k in actual: if (isinstance(v, six.string_types) or @@ -134,7 +137,6 @@ class AmuletUtils(object): def validate_relation_data(self, sentry_unit, relation, expected): """Validate actual relation data based on expected relation data.""" actual = sentry_unit.relation(relation[0], relation[1]) - self.log.debug('actual: {}'.format(repr(actual))) return self._validate_dict_data(expected, actual) def _validate_list_data(self, expected, actual): diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..11d49a7c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + base_charms = ['mysql', 'mongodb'] if self.stable: for svc in other_services: @@ -100,12 +101,37 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, + self.utopic_juno, self.vivid_kilo) = range(10) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, + ('utopic', None): self.utopic_juno, + ('vivid', None): self.vivid_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series]