openstack/charm-glance
Code Issues Proposed changes

Enable Memcache service for Token Caching

Browse Source 
With the release of 4.2.0 of keystonemiddleware using the
in-process token cache is no longer recommended. It is recommended
that a memcache backend to store tokens is used instead,

This installs and configures memcache and configures glance
to use memcache for token caching.

http://docs.openstack.org/releasenotes/keystonemiddleware/mitaka.html#id2

Change-Id: If1a95fbd4e5771eaad937573dd4ee28750080218
This commit is contained in:
Liam Young 2016-12-13 13:52:09 +00:00
parent 6c7edc8221
commit 121ee44911
12 changed files with 307 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
msg = 'No message retrieved.'
amulet.raise_status(amulet.FAIL, msg)
def validate_memcache(self, sentry_unit, conf, os_release,
earliest_release=5, section='keystone_authtoken',
check_kvs=None):
"""Check Memcache is running and is configured to be used
Example call from Amulet test:
def test_110_memcache(self):
u.validate_memcache(self.neutron_api_sentry,
'/etc/neutron/neutron.conf',
self._get_openstack_release())
:param sentry_unit: sentry unit
:param conf: OpenStack config file to check memcache settings
:param os_release: Current OpenStack release int code
:param earliest_release: Earliest Openstack release to check int code
:param section: OpenStack config file section to check
:param check_kvs: Dict of settings to check in config file
:returns: None
"""
if os_release < earliest_release:
self.log.debug('Skipping memcache checks for deployment. {} <'
'mitaka'.format(os_release))
return
_kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
self.log.debug('Checking memcached is running')
ret = self.validate_services_by_name({sentry_unit: ['memcached']})
if ret:
amulet.raise_status(amulet.FAIL, msg='Memcache running check'
'failed {}'.format(ret))
else:
self.log.debug('OK')
self.log.debug('Checking memcache url is configured in {}'.format(
conf))
if self.validate_config_data(sentry_unit, conf, section, _kvs):
message = "Memcache config error in: {}".format(conf)
amulet.raise_status(amulet.FAIL, msg=message)
else:
self.log.debug('OK')
self.log.debug('Checking memcache configuration in '
'/etc/memcached.conf')
contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
fatal=True)
ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
if ubuntu_release <= 'trusty':
memcache_listen_addr = 'ip6-localhost'
else:
memcache_listen_addr = '::1'
expected = {
'-p': '11211',
'-l': memcache_listen_addr}
found = []
for key, value in expected.items():
for line in contents.split('\n'):
if line.startswith(key):
self.log.debug('Checking {} is set to {}'.format(
key,
value))
assert value == line.split()[-1]
self.log.debug(line.split()[-1])
found.append(key)
if sorted(found) == sorted(expected.keys()):
self.log.debug('OK')
else:
message = "Memcache config error in: /etc/memcached.conf"
amulet.raise_status(amulet.FAIL, msg=message)

25
charmhelpers/contrib/openstack/context.py
View File

@ -90,6 +90,7 @@ from charmhelpers.contrib.network.ip import (
from charmhelpers.contrib.openstack.utils import (
config_flags_parser,
get_host_ip,
enable_memcache,
)
from charmhelpers.core.unitdata import kv
@ -1512,3 +1513,27 @@ class AppArmorContext(OSContextGenerator):
"".format(self.ctxt['aa_profile'],
self.ctxt['aa_profile_mode']))
raise e
class MemcacheContext(OSContextGenerator):
"""Memcache context
This context provides options for configuring a local memcache client and
server
"""
def __call__(self):
ctxt = {}
ctxt['use_memcache'] = enable_memcache(config('openstack-origin'))
if ctxt['use_memcache']:
# Trusty version of memcached does not support ::1 as a listen
# address so use host file entry instead
if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty':
ctxt['memcache_server'] = '::1'
else:
ctxt['memcache_server'] = 'ip6-localhost'
ctxt['memcache_server_formatted'] = '[::1]'
ctxt['memcache_port'] = '11211'
ctxt['memcache_url'] = 'inet6:{}:{}'.format(
ctxt['memcache_server_formatted'],
ctxt['memcache_port'])
return ctxt

53
charmhelpers/contrib/openstack/templates/memcached.conf Normal file
View File

@ -0,0 +1,53 @@
###############################################################################
# [ WARNING ]
# memcached configuration file maintained by Juju
# local changes may be overwritten.
###############################################################################
# memcached default config file
# 2003 - Jay Bonci <jaybonci@debian.org>
# This configuration file is read by the start-memcached script provided as
# part of the Debian GNU/Linux distribution.
# Run memcached as a daemon. This command is implied, and is not needed for the
# daemon to run. See the README.Debian that comes with this package for more
# information.
-d
# Log memcached's output to /var/log/memcached
logfile /var/log/memcached.log
# Be verbose
# -v
# Be even more verbose (print client commands as well)
# -vv
# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
# Note that the daemon will grow to this size, but does not start out holding this much
# memory
-m 64
# Default connection port is 11211
-p {{ memcache_port }}
# Run the daemon as root. The start-memcached will default to running as root if no
# -u command is present in this config file
-u memcache
# Specify which IP address to listen on. The default is to listen on all IP addresses
# This parameter is one of the only security measures that memcached has, so make sure
# it's listening on a firewalled interface.
-l {{ memcache_server }}
# Limit the number of simultaneous incoming connections. The daemon default is 1024
# -c 1024
# Lock down all paged memory. Consult with the README and homepage before you do this
# -k
# Return error when memory is exhausted (rather than removing items)
# -M
# Maximize core file limit
# -r

3
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
View File

@ -14,4 +14,7 @@ project_name = {{ admin_tenant_name }}
username = {{ admin_user }}
password = {{ admin_password }}
signing_dir = {{ signing_dir }}
{% if use_memcache == true %}
memcached_servers = {{ memcache_url }}
{% endif -%}
{% endif -%}

25
charmhelpers/contrib/openstack/utils.py
View File

@ -1925,3 +1925,28 @@ def os_application_version_set(package):
application_version_set(os_release(package))
else:
application_version_set(application_version)
def enable_memcache(source=None, release=None):
"""Determine if memcache should be enabled on the local unit
@param source: source string for charm
@param release: release of OpenStack currently deployed
@returns boolean Whether memcache should be enabled
"""
if not release:
release = get_os_codename_install_source(source)
return release >= 'mitaka'
def token_cache_pkgs(source=None, release=None):
"""Determine additional packages needed for token caching
@param source: source string for charm
@param release: release of OpenStack currently deployed
@returns List of package to enable token caching
"""
packages = []
if enable_memcache(source=source, release=release):
packages.extend(['memcached', 'python-memcache'])
return packages

16
hooks/glance_utils.py
View File

@ -82,6 +82,8 @@ from charmhelpers.contrib.openstack.utils import (
resume_unit,
incomplete_relation_data,
os_application_version_set,
token_cache_pkgs,
enable_memcache,
)
from charmhelpers.core.templating import render
@ -144,6 +146,7 @@ HAPROXY_CONF = "/etc/haproxy/haproxy.cfg"
HTTPS_APACHE_CONF = "/etc/apache2/sites-available/openstack_https_frontend"
HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \
"openstack_https_frontend.conf"
MEMCACHED_CONF = '/etc/memcached.conf'
TEMPLATES = 'templates/'
@ -171,7 +174,8 @@ CONFIG_FILES = OrderedDict([
context.WorkerConfigContext(),
context.OSConfigFlagContext(
charm_flag='registry-config-flags',
template_flag='registry_config_flags')],
template_flag='registry_config_flags'),
context.MemcacheContext()],
'services': ['glance-registry']
}),
(GLANCE_API_CONF, {
@ -197,7 +201,8 @@ CONFIG_FILES = OrderedDict([
context.SubordinateConfigContext(
interface=['storage-backend'],
service=['glance-api'],
config_file=GLANCE_API_CONF)],
config_file=GLANCE_API_CONF),
context.MemcacheContext()],
'services': ['glance-api']
}),
(ceph_config_file(), {
@ -256,6 +261,8 @@ def register_configs():
configs.register(HTTPS_APACHE_CONF,
CONFIG_FILES[HTTPS_APACHE_CONF]['hook_contexts'])
if enable_memcache(release=release):
configs.register(MEMCACHED_CONF, [context.MemcacheContext()])
return configs
@ -266,6 +273,7 @@ def determine_packages():
packages |= set(BASE_GIT_PACKAGES)
packages -= set(GIT_PACKAGE_BLACKLIST)
packages |= set(token_cache_pkgs(source=config('openstack-origin')))
return sorted(packages)
@ -328,6 +336,10 @@ def restart_map():
svcs.append(svc)
if svcs:
_map.append((f, svcs))
if enable_memcache(source=config('openstack-origin')):
_map.append((MEMCACHED_CONF, ['memcached']))
return OrderedDict(_map)

10
tests/basic_deployment.py
View File

@ -251,6 +251,16 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
if ret:
amulet.raise_status(amulet.FAIL, msg=ret)
def test_115_memcache(self):
u.validate_memcache(self.glance_sentry,
'/etc/glance/glance-api.conf',
self._get_openstack_release(),
earliest_release=self.trusty_mitaka)
u.validate_memcache(self.glance_sentry,
'/etc/glance/glance-registry.conf',
self._get_openstack_release(),
earliest_release=self.trusty_mitaka)
def test_200_mysql_glance_db_relation(self):
"""Verify the mysql:glance shared-db relation data"""
u.log.debug('Checking mysql to glance shared-db relation data...')

3
tests/charmhelpers/contrib/amulet/utils.py
View File

@ -148,7 +148,8 @@ class AmuletUtils(object):
for service_name in services_list:
if (self.ubuntu_releases.index(release) >= systemd_switch or
service_name in ['rabbitmq-server', 'apache2']):
service_name in ['rabbitmq-server', 'apache2',
'memcached']):
# init is systemd (or regular sysv)
cmd = 'sudo service {} status'.format(service_name)
output, code = sentry_unit.run(cmd)

67
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
msg = 'No message retrieved.'
amulet.raise_status(amulet.FAIL, msg)
def validate_memcache(self, sentry_unit, conf, os_release,
earliest_release=5, section='keystone_authtoken',
check_kvs=None):
"""Check Memcache is running and is configured to be used
Example call from Amulet test:
def test_110_memcache(self):
u.validate_memcache(self.neutron_api_sentry,
'/etc/neutron/neutron.conf',
self._get_openstack_release())
:param sentry_unit: sentry unit
:param conf: OpenStack config file to check memcache settings
:param os_release: Current OpenStack release int code
:param earliest_release: Earliest Openstack release to check int code
:param section: OpenStack config file section to check
:param check_kvs: Dict of settings to check in config file
:returns: None
"""
if os_release < earliest_release:
self.log.debug('Skipping memcache checks for deployment. {} <'
'mitaka'.format(os_release))
return
_kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
self.log.debug('Checking memcached is running')
ret = self.validate_services_by_name({sentry_unit: ['memcached']})
if ret:
amulet.raise_status(amulet.FAIL, msg='Memcache running check'
'failed {}'.format(ret))
else:
self.log.debug('OK')
self.log.debug('Checking memcache url is configured in {}'.format(
conf))
if self.validate_config_data(sentry_unit, conf, section, _kvs):
message = "Memcache config error in: {}".format(conf)
amulet.raise_status(amulet.FAIL, msg=message)
else:
self.log.debug('OK')
self.log.debug('Checking memcache configuration in '
'/etc/memcached.conf')
contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
fatal=True)
ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
if ubuntu_release <= 'trusty':
memcache_listen_addr = 'ip6-localhost'
else:
memcache_listen_addr = '::1'
expected = {
'-p': '11211',
'-l': memcache_listen_addr}
found = []
for key, value in expected.items():
for line in contents.split('\n'):
if line.startswith(key):
self.log.debug('Checking {} is set to {}'.format(
key,
value))
assert value == line.split()[-1]
self.log.debug(line.split()[-1])
found.append(key)
if sorted(found) == sorted(expected.keys()):
self.log.debug('OK')
else:
message = "Memcache config error in: /etc/memcached.conf"
amulet.raise_status(amulet.FAIL, msg=message)

3
unit_tests/test_actions_git_reinstall.py
View File

@ -31,7 +31,8 @@ with patch('actions.hooks.charmhelpers.contrib.hardening.harden.harden') as \
lambda *args, **kwargs: f(*args, **kwargs))
with patch('actions.hooks.charmhelpers.core.hookenv.config') as config:
with patch('actions.hooks.glance_utils.register_configs'):
from actions import git_reinstall
with patch('actions.hooks.glance_utils.restart_map'):
from actions import git_reinstall
TO_PATCH = [
'config',

23
unit_tests/test_glance_relations.py
View File

@ -109,8 +109,11 @@ class GlanceRelationTests(CharmTestCase):
self.config.side_effect = self.test_config.get
self.network_get_primary_address.side_effect = NotImplementedError
@patch.object(utils, 'config')
@patch.object(utils, 'token_cache_pkgs')
@patch.object(utils, 'git_install_requested')
def test_install_hook(self, git_requested):
def test_install_hook(self, git_requested, token_cache_pkgs, util_config):
token_cache_pkgs.return_value = ['memcached']
git_requested.return_value = False
repo = 'cloud:precise-grizzly'
self.test_config.set('openstack-origin', repo)
@ -119,14 +122,18 @@ class GlanceRelationTests(CharmTestCase):
self.configure_installation_source.assert_called_with(repo)
self.apt_update.assert_called_with(fatal=True)
self.apt_install.assert_called_with(
['apache2', 'glance', 'haproxy', 'python-keystone',
['apache2', 'glance', 'haproxy', 'memcached', 'python-keystone',
'python-mysqldb', 'python-psycopg2', 'python-six',
'python-swiftclient', 'uuid'], fatal=True)
self.assertTrue(self.execd_preinstall.called)
self.git_install.assert_called_with(None)
@patch.object(utils, 'config')
@patch.object(utils, 'token_cache_pkgs')
@patch.object(utils, 'git_install_requested')
def test_install_hook_precise_distro(self, git_requested):
def test_install_hook_precise_distro(self, git_requested, token_cache_pkgs,
util_config):
token_cache_pkgs.return_value = []
git_requested.return_value = False
self.test_config.set('openstack-origin', 'distro')
self.lsb_release.return_value = {'DISTRIB_RELEASE': 12.04,
@ -137,8 +144,11 @@ class GlanceRelationTests(CharmTestCase):
"cloud:precise-folsom"
)
@patch.object(utils, 'config')
@patch.object(utils, 'token_cache_pkgs')
@patch.object(utils, 'git_install_requested')
def test_install_hook_git(self, git_requested):
def test_install_hook_git(self, git_requested, token_cache_pkgs,
util_config):
git_requested.return_value = True
repo = 'cloud:trusty-juno'
openstack_origin_git = {
@ -702,9 +712,12 @@ class GlanceRelationTests(CharmTestCase):
call('/etc/haproxy/haproxy.cfg')],
configs.write.call_args_list)
@patch.object(utils, 'config')
@patch.object(utils, 'token_cache_pkgs')
@patch.object(relations, 'CONFIGS')
@patch.object(utils, 'git_install_requested')
def test_upgrade_charm(self, git_requested, configs):
def test_upgrade_charm(self, git_requested, configs, token_cache_pkgs,
util_config):
git_requested.return_value = False
self.filter_installed_packages.return_value = ['test']
relations.upgrade_charm()

26
unit_tests/test_glance_utils.py
View File

@ -49,6 +49,8 @@ TO_PATCH = [
'install_alternative',
'lsb_release',
'os_application_version_set',
'enable_memcache',
'token_cache_pkgs',
]
DPKG_OPTS = [
@ -132,6 +134,8 @@ class TestGlanceUtils(CharmTestCase):
self.mkdir.assert_called_with('/etc/ceph')
def test_restart_map(self):
self.enable_memcache.return_value = True
self.config.side_effect = None
self.service_name.return_value = 'glance'
ex_map = OrderedDict([
@ -140,19 +144,31 @@ class TestGlanceUtils(CharmTestCase):
(utils.ceph_config_file(), ['glance-api', 'glance-registry']),
(utils.HAPROXY_CONF, ['haproxy']),
(utils.HTTPS_APACHE_CONF, ['apache2']),
(utils.HTTPS_APACHE_24_CONF, ['apache2'])
(utils.HTTPS_APACHE_24_CONF, ['apache2']),
(utils.MEMCACHED_CONF, ['memcached'])
])
self.assertEquals(ex_map, utils.restart_map())
self.enable_memcache.return_value = False
del ex_map[utils.MEMCACHED_CONF]
self.assertEquals(ex_map, utils.restart_map())
@patch.object(utils, 'token_cache_pkgs')
@patch.object(utils, 'git_install_requested')
def test_determine_packages(self, git_install_requested):
def test_determine_packages(self, git_install_requested, token_cache_pkgs):
self.config.side_effect = None
token_cache_pkgs.return_value = []
git_install_requested.return_value = False
result = utils.determine_packages()
ex = utils.PACKAGES
self.assertEquals(set(ex), set(result))
self.assertEquals(set(ex), set(utils.determine_packages()))
token_cache_pkgs.return_value = ['memcached']
ex.append('memcached')
self.assertEquals(set(ex), set(utils.determine_packages()))
@patch.object(utils, 'token_cache_pkgs')
@patch.object(utils, 'git_install_requested')
def test_determine_packages_git(self, git_install_requested):
def test_determine_packages_git(self, git_install_requested,
token_cache_pkgs):
self.config.side_effect = None
git_install_requested.return_value = True
result = utils.determine_packages()
ex = utils.PACKAGES + utils.BASE_GIT_PACKAGES