[beisner,r=james-page] auto normalize amulet test definitions and amulet make targets; charm-helper sync.

Makefile

@@ -13,8 +13,7 @@ test:
 	# coreycb note: The -v should only be temporary until Amulet sends
 	# raise_status() messages to stderr:
 	#   https://bugs.launchpad.net/amulet/+bug/1320357
-	@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
+	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
-        00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

hooks/charmhelpers/contrib/network/ip.py

@@ -17,13 +17,16 @@
 import glob
 import re
 import subprocess
+import six
+import socket
 
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    log
+    log,
+    WARNING,
 )
 
 try:
@@ -365,3 +368,83 @@ def is_bridge_member(nic):
             return True
 
     return False
+
+
+def is_ip(address):
+    """
+    Returns True if address is a valid IP address.
+    """
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(address)
+        return True
+    except socket.error:
+        return False
+
+
+def ns_query(address):
+    try:
+        import dns.resolver
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.resolver
+
+    if isinstance(address, dns.name.Name):
+        rtype = 'PTR'
+    elif isinstance(address, six.string_types):
+        rtype = 'A'
+    else:
+        return None
+
+    answers = dns.resolver.query(address, rtype)
+    if answers:
+        return str(answers[0])
+    return None
+
+
+def get_host_ip(hostname, fallback=None):
+    """
+    Resolves the IP for a given hostname, or returns
+    the input if it is already an IP.
+    """
+    if is_ip(hostname):
+        return hostname
+
+    ip_addr = ns_query(hostname)
+    if not ip_addr:
+        try:
+            ip_addr = socket.gethostbyname(hostname)
+        except:
+            log("Failed to resolve hostname '%s'" % (hostname),
+                level=WARNING)
+            return fallback
+    return ip_addr
+
+
+def get_hostname(address, fqdn=True):
+    """
+    Resolves hostname for given IP, or returns the input
+    if it is already a hostname.
+    """
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install("python-dnspython")
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
+
+    if fqdn:
+        # strip trailing .
+        if result.endswith('.'):
+            return result[:-1]
+        else:
+            return result
+    else:
+        return result.split('.')[0]

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+        base_charms = ['mysql', 'mongodb']
 
         if self.stable:
             for svc in other_services:
@@ -100,12 +101,37 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo,
+         self.utopic_juno, self.vivid_kilo) = range(10)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

hooks/charmhelpers/contrib/openstack/context.py

@@ -16,11 +16,13 @@
 
 import json
 import os
+import re
 import time
 from base64 import b64decode
 from subprocess import check_call
 
 import six
+import yaml
 
 from charmhelpers.fetch import (
     apt_install,
@@ -45,8 +47,11 @@ from charmhelpers.core.hookenv import (
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
+from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    list_nics,
+    get_nic_hwaddr,
     mkdir,
     write_file,
 )
@@ -63,16 +68,22 @@ from charmhelpers.contrib.hahelpers.apache import (
 )
 from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
+    parse_data_port_mappings,
+)
+from charmhelpers.contrib.openstack.ip import (
+    resolve_address,
+    INTERNAL,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
+    get_ipv4_addr,
     get_ipv6_addr,
     get_netmask_for_address,
     format_ipv6_addr,
     is_address_in_network,
+    is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
-
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -104,9 +115,41 @@ def context_complete(ctxt):
 def config_flags_parser(config_flags):
     """Parses config flags string into dict.
 
+    This parsing method supports a few different formats for the config
+    flag values to be parsed:
+
+      1. A string in the simple format of key=value pairs, with the possibility
+         of specifying multiple key value pairs within the same string. For
+         example, a string in the format of 'key1=value1, key2=value2' will
+         return a dict of:
+         {'key1': 'value1',
+          'key2': 'value2'}.
+
+      2. A string in the above format, but supporting a comma-delimited list
+         of values for the same key. For example, a string in the format of
+         'key1=value1, key2=value3,value4,value5' will return a dict of:
+         {'key1', 'value1',
+          'key2', 'value2,value3,value4'}
+
+      3. A string containing a colon character (:) prior to an equal
+         character (=) will be treated as yaml and parsed as such. This can be
+         used to specify more complex key value pairs. For example,
+         a string in the format of 'key1: subkey1=value1, subkey2=value2' will
+         return a dict of:
+         {'key1', 'subkey1=value1, subkey2=value2'}
+
     The provided config_flags string may be a list of comma-separated values
     which themselves may be comma-separated list of values.
     """
+    # If we find a colon before an equals sign then treat it as yaml.
+    # Note: limit it to finding the colon first since this indicates assignment
+    # for inline yaml.
+    colon = config_flags.find(':')
+    equals = config_flags.find('=')
+    if colon > 0:
+        if colon < equals or equals < 0:
+            return yaml.safe_load(config_flags)
+
     if config_flags.find('==') >= 0:
         log("config_flags is not in expected format (key=value)", level=ERROR)
         raise OSContextError
@@ -191,7 +234,7 @@ class SharedDBContext(OSContextGenerator):
                                         unit=local_unit())
             if set_hostname != access_hostname:
                 relation_set(relation_settings={hostname_key: access_hostname})
-                return ctxt  # Defer any further hook execution for now....
+                return None  # Defer any further hook execution for now....
 
         password_setting = 'password'
         if self.relation_prefix:
@@ -277,12 +320,29 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 
 class IdentityServiceContext(OSContextGenerator):
-    interfaces = ['identity-service']
+
+    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
+        self.service = service
+        self.service_user = service_user
+        self.rel_name = rel_name
+        self.interfaces = [self.rel_name]
 
     def __call__(self):
-        log('Generating template context for identity-service', level=DEBUG)
+        log('Generating template context for ' + self.rel_name, level=DEBUG)
         ctxt = {}
-        for rid in relation_ids('identity-service'):
+
+        if self.service and self.service_user:
+            # This is required for pki token signing if we don't want /tmp to
+            # be used.
+            cachedir = '/var/cache/%s' % (self.service)
+            if not os.path.isdir(cachedir):
+                log("Creating service cache dir %s" % (cachedir), level=DEBUG)
+                mkdir(path=cachedir, owner=self.service_user,
+                      group=self.service_user, perms=0o700)
+
+            ctxt['signing_dir'] = cachedir
+
+        for rid in relation_ids(self.rel_name):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -291,15 +351,16 @@ class IdentityServiceContext(OSContextGenerator):
                 auth_host = format_ipv6_addr(auth_host) or auth_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
-                ctxt = {'service_port': rdata.get('service_port'),
+                ctxt.update({'service_port': rdata.get('service_port'),
-                        'service_host': serv_host,
+                             'service_host': serv_host,
-                        'auth_host': auth_host,
+                             'auth_host': auth_host,
-                        'auth_port': rdata.get('auth_port'),
+                             'auth_port': rdata.get('auth_port'),
-                        'admin_tenant_name': rdata.get('service_tenant'),
+                             'admin_tenant_name': rdata.get('service_tenant'),
-                        'admin_user': rdata.get('service_username'),
+                             'admin_user': rdata.get('service_username'),
-                        'admin_password': rdata.get('service_password'),
+                             'admin_password': rdata.get('service_password'),
-                        'service_protocol': svc_protocol,
+                             'service_protocol': svc_protocol,
-                        'auth_protocol': auth_protocol}
+                             'auth_protocol': auth_protocol})
+
                 if context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
@@ -398,6 +459,11 @@ class AMQPContext(OSContextGenerator):
 
                 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
 
+        oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
+        if oslo_messaging_flags:
+            ctxt['oslo_messaging_flags'] = config_flags_parser(
+                oslo_messaging_flags)
+
         if not context_complete(ctxt):
             return {}
 
@@ -677,7 +743,14 @@ class ApacheSSLContext(OSContextGenerator):
                 'endpoints': [],
                 'ext_ports': []}
 
-        for cn in self.canonical_names():
+        cns = self.canonical_names()
+        if cns:
+            for cn in cns:
+                self.configure_cert(cn)
+        else:
+            # Expect cert/key provided in config (currently assumed that ca
+            # uses ip for cn)
+            cn = resolve_address(endpoint_type=INTERNAL)
             self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
@@ -740,6 +813,19 @@ class NeutronContext(OSContextGenerator):
 
         return ovs_ctxt
 
+    def nuage_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        nuage_ctxt = {'core_plugin': driver,
+                      'neutron_plugin': 'vsp',
+                      'neutron_security_groups': self.neutron_security_groups,
+                      'local_ip': unit_private_ip(),
+                      'config': config}
+
+        return nuage_ctxt
+
     def nvp_ctxt(self):
         driver = neutron_plugin_attribute(self.plugin, 'driver',
                                           self.network_manager)
@@ -823,6 +909,8 @@ class NeutronContext(OSContextGenerator):
             ctxt.update(self.n1kv_ctxt())
         elif self.plugin == 'Calico':
             ctxt.update(self.calico_ctxt())
+        elif self.plugin == 'vsp':
+            ctxt.update(self.nuage_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -833,6 +921,48 @@ class NeutronContext(OSContextGenerator):
         return ctxt
 
 
+class NeutronPortContext(OSContextGenerator):
+    NIC_PREFIXES = ['eth', 'bond']
+
+    def resolve_ports(self, ports):
+        """Resolve NICs not yet bound to bridge(s)
+
+        If hwaddress provided then returns resolved hwaddress otherwise NIC.
+        """
+        if not ports:
+            return None
+
+        hwaddr_to_nic = {}
+        hwaddr_to_ip = {}
+        for nic in list_nics(self.NIC_PREFIXES):
+            hwaddr = get_nic_hwaddr(nic)
+            hwaddr_to_nic[hwaddr] = nic
+            addresses = get_ipv4_addr(nic, fatal=False)
+            addresses += get_ipv6_addr(iface=nic, fatal=False)
+            hwaddr_to_ip[hwaddr] = addresses
+
+        resolved = []
+        mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
+        for entry in ports:
+            if re.match(mac_regex, entry):
+                # NIC is in known NICs and does NOT hace an IP address
+                if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]:
+                    # If the nic is part of a bridge then don't use it
+                    if is_bridge_member(hwaddr_to_nic[entry]):
+                        continue
+
+                    # Entry is a MAC address for a valid interface that doesn't
+                    # have an IP address assigned yet.
+                    resolved.append(hwaddr_to_nic[entry])
+            else:
+                # If the passed entry is not a MAC address, assume it's a valid
+                # interface, and that the user put it there on purpose (we can
+                # trust it to be the real external network).
+                resolved.append(entry)
+
+        return resolved
+
+
 class OSConfigFlagContext(OSContextGenerator):
     """Provides support for user-defined config flags.
 
@@ -1021,6 +1151,8 @@ class ZeroMQContext(OSContextGenerator):
                     for unit in related_units(rid):
                         ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
                         ctxt['zmq_host'] = relation_get('host', unit, rid)
+                        ctxt['zmq_redis_address'] = relation_get(
+                            'zmq_redis_address', unit, rid)
 
         return ctxt
 
@@ -1052,3 +1184,145 @@ class SysctlContext(OSContextGenerator):
             sysctl_create(sysctl_dict,
                           '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
         return {'sysctl': sysctl_dict}
+
+
+class NeutronAPIContext(OSContextGenerator):
+    '''
+    Inspects current neutron-plugin-api relation for neutron settings. Return
+    defaults if it is not present.
+    '''
+    interfaces = ['neutron-plugin-api']
+
+    def __call__(self):
+        self.neutron_defaults = {
+            'l2_population': {
+                'rel_key': 'l2-population',
+                'default': False,
+            },
+            'overlay_network_type': {
+                'rel_key': 'overlay-network-type',
+                'default': 'gre',
+            },
+            'neutron_security_groups': {
+                'rel_key': 'neutron-security-groups',
+                'default': False,
+            },
+            'network_device_mtu': {
+                'rel_key': 'network-device-mtu',
+                'default': None,
+            },
+            'enable_dvr': {
+                'rel_key': 'enable-dvr',
+                'default': False,
+            },
+            'enable_l3ha': {
+                'rel_key': 'enable-l3ha',
+                'default': False,
+            },
+        }
+        ctxt = self.get_neutron_options({})
+        for rid in relation_ids('neutron-plugin-api'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                if 'l2-population' in rdata:
+                    ctxt.update(self.get_neutron_options(rdata))
+
+        return ctxt
+
+    def get_neutron_options(self, rdata):
+        settings = {}
+        for nkey in self.neutron_defaults.keys():
+            defv = self.neutron_defaults[nkey]['default']
+            rkey = self.neutron_defaults[nkey]['rel_key']
+            if rkey in rdata.keys():
+                if type(defv) is bool:
+                    settings[nkey] = bool_from_string(rdata[rkey])
+                else:
+                    settings[nkey] = rdata[rkey]
+            else:
+                settings[nkey] = defv
+        return settings
+
+
+class ExternalPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        ports = config('ext-port')
+        if ports:
+            ports = [p.strip() for p in ports.split()]
+            ports = self.resolve_ports(ports)
+            if ports:
+                ctxt = {"ext_port": ports[0]}
+                napi_settings = NeutronAPIContext()()
+                mtu = napi_settings.get('network_device_mtu')
+                if mtu:
+                    ctxt['ext_port_mtu'] = mtu
+
+        return ctxt
+
+
+class DataPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ports = config('data-port')
+        if ports:
+            portmap = parse_data_port_mappings(ports)
+            ports = portmap.values()
+            resolved = self.resolve_ports(ports)
+            normalized = {get_nic_hwaddr(port): port for port in resolved
+                          if port not in ports}
+            normalized.update({port: port for port in resolved
+                               if port in ports})
+            if resolved:
+                return {bridge: normalized[port] for bridge, port in
+                        six.iteritems(portmap) if port in normalized.keys()}
+
+        return None
+
+
+class PhyNICMTUContext(DataPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        mappings = super(PhyNICMTUContext, self).__call__()
+        if mappings and mappings.values():
+            ports = mappings.values()
+            napi_settings = NeutronAPIContext()()
+            mtu = napi_settings.get('network_device_mtu')
+            if mtu:
+                ctxt["devs"] = '\\n'.join(ports)
+                ctxt['mtu'] = mtu
+
+        return ctxt
+
+
+class NetworkServiceContext(OSContextGenerator):
+
+    def __init__(self, rel_name='quantum-network-service'):
+        self.rel_name = rel_name
+        self.interfaces = [rel_name]
+
+    def __call__(self):
+        for rid in relation_ids(self.rel_name):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                ctxt = {
+                    'keystone_host': rdata.get('keystone_host'),
+                    'service_port': rdata.get('service_port'),
+                    'auth_port': rdata.get('auth_port'),
+                    'service_tenant': rdata.get('service_tenant'),
+                    'service_username': rdata.get('service_username'),
+                    'service_password': rdata.get('service_password'),
+                    'quantum_host': rdata.get('quantum_host'),
+                    'quantum_port': rdata.get('quantum_port'),
+                    'quantum_url': rdata.get('quantum_url'),
+                    'region': rdata.get('region'),
+                    'service_protocol':
+                    rdata.get('service_protocol') or 'http',
+                    'auth_protocol':
+                    rdata.get('auth_protocol') or 'http',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -16,6 +16,7 @@
 
 # Various utilies for dealing with Neutron and the renaming from Quantum.
 
+import six
 from subprocess import check_output
 
 from charmhelpers.core.hookenv import (
@@ -179,6 +180,19 @@ def neutron_plugins():
                           'nova-api-metadata']],
             'server_packages': ['neutron-server', 'calico-control'],
             'server_services': ['neutron-server']
+        },
+        'vsp': {
+            'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
+            'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [],
+            'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':
@@ -237,3 +251,72 @@ def network_manager():
     else:
         # ensure accurate naming for all releases post-H
         return 'neutron'
+
+
+def parse_mappings(mappings):
+    parsed = {}
+    if mappings:
+        mappings = mappings.split(' ')
+        for m in mappings:
+            p = m.partition(':')
+            if p[1] == ':':
+                parsed[p[0].strip()] = p[2].strip()
+
+    return parsed
+
+
+def parse_bridge_mappings(mappings):
+    """Parse bridge mappings.
+
+    Mappings must be a space-delimited list of provider:bridge mappings.
+
+    Returns dict of the form {provider:bridge}.
+    """
+    return parse_mappings(mappings)
+
+
+def parse_data_port_mappings(mappings, default_bridge='br-data'):
+    """Parse data port mappings.
+
+    Mappings must be a space-delimited list of bridge:port mappings.
+
+    Returns dict of the form {bridge:port}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        if not mappings:
+            return {}
+
+        # For backwards-compatibility we need to support port-only provided in
+        # config.
+        _mappings = {default_bridge: mappings.split(' ')[0]}
+
+    bridges = _mappings.keys()
+    ports = _mappings.values()
+    if len(set(bridges)) != len(bridges):
+        raise Exception("It is not allowed to have more than one port "
+                        "configured on the same bridge")
+
+    if len(set(ports)) != len(ports):
+        raise Exception("It is not allowed to have the same port configured "
+                        "on more than one bridge")
+
+    return _mappings
+
+
+def parse_vlan_range_mappings(mappings):
+    """Parse vlan range mappings.
+
+    Mappings must be a space-delimited list of provider:start:end mappings.
+
+    Returns dict of the form {provider: (start, end)}.
+    """
+    _mappings = parse_mappings(mappings)
+    if not _mappings:
+        return {}
+
+    mappings = {}
+    for p, r in six.iteritems(_mappings):
+        mappings[p] = tuple(r.split(':'))
+
+    return mappings

hooks/charmhelpers/contrib/openstack/templates/git.upstart

@@ -0,0 +1,17 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+            --chdir {{ start_dir }} --name {{ process_name }} \
+            --exec {{ executable_name }} -- \
+            {% for config_file in config_files -%}
+            --config-file={{ config_file }} \
+            {% endfor -%}
+            {% if log_file -%}
+            --log-file={{ log_file }}
+            {% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -0,0 +1,9 @@
+{% if auth_host -%}
+[keystone_authtoken]
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo

@@ -0,0 +1,22 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/templates/section-zeromq

@@ -0,0 +1,14 @@
+{% if zmq_host -%}
+# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
+rpc_backend = zmq
+rpc_zmq_host = {{ zmq_host }}
+{% if zmq_redis_address -%}
+rpc_zmq_matchmaker = redis
+matchmaker_heartbeat_freq = 15
+matchmaker_heartbeat_ttl = 30
+[matchmaker_redis]
+host = {{ zmq_redis_address }}
+{% else -%}
+rpc_zmq_matchmaker = ring
+{% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -23,12 +23,17 @@ from functools import wraps
 import subprocess
 import json
 import os
-import socket
 import sys
 
 import six
 import yaml
 
+from charmhelpers.contrib.network import ip
+
+from charmhelpers.core import (
+    unitdata,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -329,6 +334,21 @@ def configure_installation_source(rel):
         error_out("Invalid openstack-release specified: %s" % rel)
 
 
+def config_value_changed(option):
+    """
+    Determine if config value changed since last call to this function.
+    """
+    hook_data = unitdata.HookData()
+    with hook_data():
+        db = unitdata.kv()
+        current = config(option)
+        saved = db.get(option)
+        db.set(option, current)
+        if saved is None:
+            return False
+        return current != saved
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing
@@ -421,77 +441,10 @@ def clean_storage(block_device):
     else:
         zap_disk(block_device)
 
-
+is_ip = ip.is_ip
-def is_ip(address):
+ns_query = ip.ns_query
-    """
+get_host_ip = ip.get_host_ip
-    Returns True if address is a valid IP address.
+get_hostname = ip.get_hostname
-    """
-    try:
-        # Test to see if already an IPv4 address
-        socket.inet_aton(address)
-        return True
-    except socket.error:
-        return False
-
-
-def ns_query(address):
-    try:
-        import dns.resolver
-    except ImportError:
-        apt_install('python-dnspython')
-        import dns.resolver
-
-    if isinstance(address, dns.name.Name):
-        rtype = 'PTR'
-    elif isinstance(address, six.string_types):
-        rtype = 'A'
-    else:
-        return None
-
-    answers = dns.resolver.query(address, rtype)
-    if answers:
-        return str(answers[0])
-    return None
-
-
-def get_host_ip(hostname):
-    """
-    Resolves the IP for a given hostname, or returns
-    the input if it is already an IP.
-    """
-    if is_ip(hostname):
-        return hostname
-
-    return ns_query(hostname)
-
-
-def get_hostname(address, fqdn=True):
-    """
-    Resolves hostname for given IP, or returns the input
-    if it is already a hostname.
-    """
-    if is_ip(address):
-        try:
-            import dns.reversename
-        except ImportError:
-            apt_install('python-dnspython')
-            import dns.reversename
-
-        rev = dns.reversename.from_address(address)
-        result = ns_query(rev)
-        if not result:
-            return None
-    else:
-        result = address
-
-    if fqdn:
-        # strip trailing .
-        if result.endswith('.'):
-            return result[:-1]
-        else:
-            return result
-    else:
-        return result.split('.')[0]
 
 
 def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
@@ -535,82 +488,106 @@ def os_requires_version(ostack_release, pkg):
 
 
 def git_install_requested():
-    """Returns true if openstack-origin-git is specified."""
+    """
-    return config('openstack-origin-git') != "None"
+    Returns true if openstack-origin-git is specified.
+    """
+    return config('openstack-origin-git') is not None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
+def git_clone_and_install(projects_yaml, core_project):
-    """Clone/install all OpenStack repos specified in yaml config file."""
+    """
-    global requirements_dir
+    Clone/install all specified OpenStack repositories.
 
-    if file_name == "None":
+    The expected format of projects_yaml is:
+        repositories:
+          - {name: keystone,
+             repository: 'git://git.openstack.org/openstack/keystone.git',
+             branch: 'stable/icehouse'}
+          - {name: requirements,
+             repository: 'git://git.openstack.org/openstack/requirements.git',
+             branch: 'stable/icehouse'}
+        directory: /mnt/openstack-git
+        http_proxy: http://squid.internal:3128
+        https_proxy: https://squid.internal:3128
+
+        The directory, http_proxy, and https_proxy keys are optional.
+    """
+    global requirements_dir
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
+    projects = yaml.load(projects_yaml)
+    _git_validate_projects_yaml(projects, core_project)
 
-    # clone/install the requirements project first
+    old_environ = dict(os.environ)
-    installed = _git_clone_and_install_subset(yaml_file,
-                                              whitelist=['requirements'])
-    if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
 
-    # clone/install all other projects except requirements and the core project
+    if 'http_proxy' in projects.keys():
-    blacklist = ['requirements', core_project]
+        os.environ['http_proxy'] = projects['http_proxy']
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
+    if 'https_proxy' in projects.keys():
-                                  update_requirements=True)
+        os.environ['https_proxy'] = projects['https_proxy']
 
-    # clone/install the core project
+    if 'directory' in projects.keys():
-    whitelist = [core_project]
+        parent_dir = projects['directory']
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
+
-                                              update_requirements=True)
+    for p in projects['repositories']:
-    if core_project not in installed:
+        repo = p['repository']
-        error_out('{} git repository must be specified'.format(core_project))
+        branch = p['branch']
+        if p['name'] == 'requirements':
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=False)
+            requirements_dir = repo_dir
+        else:
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=True)
+
+    os.environ = old_environ
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
+def _git_validate_projects_yaml(projects, core_project):
-                                  update_requirements=False):
+    """
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
+    Validate the projects yaml.
-    global requirements_dir
+    """
-    installed = []
+    _git_ensure_key_exists('repositories', projects)
 
-    with open(yaml_file, 'r') as fd:
+    for project in projects['repositories']:
-        projects = yaml.load(fd)
+        _git_ensure_key_exists('name', project.keys())
-        for proj, val in projects.items():
+        _git_ensure_key_exists('repository', project.keys())
-            # The project subset is chosen based on the following 3 rules:
+        _git_ensure_key_exists('branch', project.keys())
-            # 1) If project is in blacklist, we don't clone/install it, period.
+
-            # 2) If whitelist is empty, we clone/install everything else.
+    if projects['repositories'][0]['name'] != 'requirements':
-            # 3) If whitelist is not empty, we clone/install everything in the
+        error_out('{} git repo must be specified first'.format('requirements'))
-            #    whitelist.
+
-            if proj in blacklist:
+    if projects['repositories'][-1]['name'] != core_project:
-                continue
+        error_out('{} git repo must be specified last'.format(core_project))
-            if whitelist and proj not in whitelist:
-                continue
-            repo = val['repository']
-            branch = val['branch']
-            repo_dir = _git_clone_and_install_single(repo, branch,
-                                                     update_requirements)
-            if proj == 'requirements':
-                requirements_dir = repo_dir
-            installed.append(proj)
-    return installed
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
+def _git_ensure_key_exists(key, keys):
-    """Clone and install a single git repository."""
+    """
-    dest_parent_dir = "/mnt/openstack-git/"
+    Ensure that key exists in keys.
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+    """
+    if key not in keys:
+        error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
-    if not os.path.exists(dest_parent_dir):
+
-        juju_log('Host dir not mounted at {}. '
+def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
-                 'Creating directory there instead.'.format(dest_parent_dir))
+    """
-        os.mkdir(dest_parent_dir)
+    Clone and install a single git repository.
+    """
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(parent_dir):
+        juju_log('Directory already exists at {}. '
+                 'No need to create directory.'.format(parent_dir))
+        os.mkdir(parent_dir)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
     else:
         repo_dir = dest_dir
 
@@ -627,16 +604,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False):
 
 
 def _git_update_requirements(package_dir, reqs_dir):
-    """Update from global requirements.
+    """
+    Update from global requirements.
 
-       Update an OpenStack git directory's requirements.txt and
+    Update an OpenStack git directory's requirements.txt and
-       test-requirements.txt from global-requirements.txt."""
+    test-requirements.txt from global-requirements.txt.
+    """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = "python update.py {}".format(package_dir)
+    cmd = ['python', 'update.py', package_dir]
     try:
-        subprocess.check_call(cmd.split(' '))
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
         error_out("Error updating {} from global-requirements.txt".format(package))
     os.chdir(orig_dir)
+
+
+def git_src_dir(projects_yaml, project):
+    """
+    Return the directory where the specified project's source is located.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    for p in projects['repositories']:
+        if p['name'] == project:
+            return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+    return None

hooks/charmhelpers/core/hookenv.py

@@ -20,11 +20,13 @@
 # Authors:
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
+from __future__ import print_function
 import os
 import json
 import yaml
 import subprocess
 import sys
+import errno
 from subprocess import CalledProcessError
 
 import six
@@ -87,7 +89,18 @@ def log(message, level=None):
     if not isinstance(message, six.string_types):
         message = repr(message)
     command += [message]
-    subprocess.call(command)
+    # Missing juju-log should not cause failures in unit tests
+    # Send log output to stderr
+    try:
+        subprocess.call(command)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            if level:
+                message = "{}: {}".format(level, message)
+            message = "juju-log: {}".format(message)
+            print(message, file=sys.stderr)
+        else:
+            raise
 
 
 class Serializable(UserDict):
@@ -566,3 +579,29 @@ class Hooks(object):
 def charm_dir():
     """Return the root directory of the current charm"""
     return os.environ.get('CHARM_DIR')
+
+
+@cached
+def action_get(key=None):
+    """Gets the value of an action parameter, or all key/value param pairs"""
+    cmd = ['action-get']
+    if key is not None:
+        cmd.append(key)
+    cmd.append('--format=json')
+    action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+    return action_data
+
+
+def action_set(values):
+    """Sets the values to be returned after the action finishes"""
+    cmd = ['action-set']
+    for k, v in list(values.items()):
+        cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)
+
+
+def action_fail(message):
+    """Sets the action status to failed and sets the error message.
+
+    The results set by action_set are preserved."""
+    subprocess.check_call(['action-fail', message])

hooks/charmhelpers/core/host.py

@@ -339,12 +339,16 @@ def lsb_release():
 def pwgen(length=None):
     """Generate a random pasword."""
     if length is None:
+        # A random length is ok to use a weak PRNG
         length = random.choice(range(35, 45))
     alphanumeric_chars = [
         l for l in (string.ascii_letters + string.digits)
         if l not in 'l0QD1vAEIOUaeiou']
+    # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
+    # actual password
+    random_generator = random.SystemRandom()
     random_chars = [
-        random.choice(alphanumeric_chars) for _ in range(length)]
+        random_generator.choice(alphanumeric_chars) for _ in range(length)]
     return(''.join(random_chars))
 
 

hooks/charmhelpers/core/services/helpers.py

@@ -45,12 +45,14 @@ class RelationContext(dict):
     """
     name = None
     interface = None
-    required_keys = []
 
     def __init__(self, name=None, additional_required_keys=None):
+        if not hasattr(self, 'required_keys'):
+            self.required_keys = []
+
         if name is not None:
             self.name = name
-        if additional_required_keys is not None:
+        if additional_required_keys:
             self.required_keys.extend(additional_required_keys)
         self.get_data()
 
@@ -134,7 +136,10 @@ class MysqlRelation(RelationContext):
     """
     name = 'db'
     interface = 'mysql'
-    required_keys = ['host', 'user', 'password', 'database']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'user', 'password', 'database']
+        RelationContext.__init__(self, *args, **kwargs)
 
 
 class HttpRelation(RelationContext):
@@ -146,7 +151,10 @@ class HttpRelation(RelationContext):
     """
     name = 'website'
     interface = 'http'
-    required_keys = ['host', 'port']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'port']
+        RelationContext.__init__(self, *args, **kwargs)
 
     def provide_data(self):
         return {

hooks/charmhelpers/core/strutils.py

@@ -33,9 +33,9 @@ def bool_from_string(value):
 
     value = value.strip().lower()
 
-    if value in ['y', 'yes', 'true', 't']:
+    if value in ['y', 'yes', 'true', 't', 'on']:
         return True
-    elif value in ['n', 'no', 'false', 'f']:
+    elif value in ['n', 'no', 'false', 'f', 'off']:
         return False
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)

hooks/charmhelpers/core/unitdata.py

@@ -443,7 +443,7 @@ class HookData(object):
         data = hookenv.execution_environment()
         self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
         self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
-        self.kv.set('env', data['env'])
+        self.kv.set('env', dict(data['env']))
         self.kv.set('unit', data['unit'])
         self.kv.set('relid', data.get('relid'))
         return conf_delta, rels_delta

tests/016-basic-trusty-juno

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic ceph-radosgw deployment on trusty-juno."""
+
+from basic_deployment import CephRadosGwBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephRadosGwBasicDeployment(series='trusty',
+                                     openstack='cloud:trusty-juno',
+                                     source='cloud:trusty-updates/juno')
+    deployment.run_tests()

tests/017-basic-trusty-kilo

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic ceph-radosgw deployment on trusty-kilo."""
+
+from basic_deployment import CephRadosGwBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephRadosGwBasicDeployment(series='trusty',
+                                     openstack='cloud:trusty-kilo',
+                                     source='cloud:trusty-updates/kilo')
+    deployment.run_tests()

tests/018-basic-utopic-juno

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic ceph-radosgw deployment on utopic-juno."""
+
+from basic_deployment import CephRadosGwBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephRadosGwBasicDeployment(series='utopic')
+    deployment.run_tests()

tests/019-basic-vivid-kilo

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic ceph-radosgw deployment on vivid-kilo."""
+
+from basic_deployment import CephRadosGwBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephRadosGwBasicDeployment(series='vivid')
+    deployment.run_tests()

tests/basic_deployment.py

@@ -11,7 +11,7 @@ from charmhelpers.contrib.openstack.amulet.utils import (  # noqa
 )
 
 # Use DEBUG to turn on debug logging
-u = OpenStackAmuletUtils(ERROR)
+u = OpenStackAmuletUtils(DEBUG)
 
 
 class CephRadosGwBasicDeployment(OpenStackAmuletDeployment):

tests/charmhelpers/contrib/amulet/utils.py

@@ -118,6 +118,9 @@ class AmuletUtils(object):
            longs, or can be a function that evaluate a variable and returns a
            bool.
            """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        self.log.debug('expected: {}'.format(repr(expected)))
+
         for k, v in six.iteritems(expected):
             if k in actual:
                 if (isinstance(v, six.string_types) or
@@ -134,7 +137,6 @@ class AmuletUtils(object):
     def validate_relation_data(self, sentry_unit, relation, expected):
         """Validate actual relation data based on expected relation data."""
         actual = sentry_unit.relation(relation[0], relation[1])
-        self.log.debug('actual: {}'.format(repr(actual)))
         return self._validate_dict_data(expected, actual)
 
     def _validate_list_data(self, expected, actual):

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -43,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+        base_charms = ['mysql', 'mongodb']
 
         if self.stable:
             for svc in other_services:
@@ -100,12 +101,37 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo,
+         self.utopic_juno, self.vivid_kilo) = range(10)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]