Introduces a keepalived manager for HA

This patch introduces a keepalived manager
which will be used for the blueprint
blueprint l3-high-availability.

The manager can create a keepalived.conf compliant configuration,
start, stop and restart the service, as well as create keepalived
notification scripts.

Implements: blueprint l3-high-availability
Change-Id: I1ba9f332778f27de950d9e97d4fb4a337f6f26da
Co-Authored-By: Assaf Muller <amuller@redhat.com>
This commit is contained in:
Assaf Muller 2014-08-05 23:47:18 +03:00
parent c04bdd07bb
commit d915552153
5 changed files with 685 additions and 7 deletions

View File

@ -42,3 +42,7 @@ iptables-save: CommandFilter, iptables-save, root
iptables-restore: CommandFilter, iptables-restore, root
ip6tables-save: CommandFilter, ip6tables-save, root
ip6tables-restore: CommandFilter, ip6tables-restore, root
# Keepalived
keepalived: CommandFilter, keepalived, root
kill_keepalived: KillFilter, root, /usr/sbin/keepalived, -HUP, -15, -9

View File

@ -37,7 +37,7 @@ class ProcessManager(object):
Note: The manager expects uuid to be in cmdline.
"""
def __init__(self, conf, uuid, root_helper='sudo',
namespace=None, service=None):
namespace=None, service=None, pids_path=None):
self.conf = conf
self.uuid = uuid
self.root_helper = root_helper
@ -46,6 +46,7 @@ class ProcessManager(object):
self.service_pid_fname = 'pid.' + service
else:
self.service_pid_fname = 'pid'
self.pids_path = pids_path or self.conf.external_pids
def enable(self, cmd_callback, reload_cfg=False):
if not self.active:
@ -67,18 +68,19 @@ class ProcessManager(object):
utils.execute(cmd, self.root_helper)
# In the case of shutting down, remove the pid file
if sig == '9':
utils.remove_conf_file(self.conf.external_pids,
utils.remove_conf_file(self.pids_path,
self.uuid,
self.service_pid_fname)
elif pid:
LOG.debug(_('Process for %(uuid)s pid %(pid)d is stale, ignoring '
'command'), {'uuid': self.uuid, 'pid': pid})
LOG.debug('Process for %(uuid)s pid %(pid)d is stale, ignoring '
'signal %(signal)s', {'uuid': self.uuid, 'pid': pid,
'signal': sig})
else:
LOG.debug(_('No process started for %s'), self.uuid)
LOG.debug('No process started for %s', self.uuid)
def get_pid_file_name(self, ensure_pids_dir=False):
"""Returns the file name for a given kind of config file."""
return utils.get_conf_file_name(self.conf.external_pids,
return utils.get_conf_file_name(self.pids_path,
self.uuid,
self.service_pid_fname,
ensure_pids_dir)
@ -86,7 +88,7 @@ class ProcessManager(object):
@property
def pid(self):
"""Last known pid for this external process spawned for this uuid."""
return utils.get_value_from_conf_file(self.conf.external_pids,
return utils.get_value_from_conf_file(self.pids_path,
self.uuid,
self.service_pid_fname,
int)

View File

@ -0,0 +1,371 @@
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import os
import stat
from oslo.config import cfg
from neutron.agent.linux import external_process
from neutron.agent.linux import utils
from neutron.common import exceptions
from neutron.openstack.common.gettextutils import _LW
from neutron.openstack.common import log as logging
VALID_STATES = ['MASTER', 'BACKUP']
VALID_NOTIFY_STATES = ['master', 'backup', 'fault']
VALID_AUTH_TYPES = ['AH', 'PASS']
HA_DEFAULT_PRIORITY = 50
LOG = logging.getLogger(__name__)
class InvalidInstanceStateException(exceptions.NeutronException):
message = (_('Invalid instance state: %%(state)s, valid states are: '
'%(valid_states)s') %
{'valid_states': ', '.join(VALID_STATES)})
class InvalidNotifyStateException(exceptions.NeutronException):
message = (_('Invalid notify state: %%(state)s, valid states are: '
'%(valid_notify_states)s') %
{'valid_notify_states': ', '.join(VALID_NOTIFY_STATES)})
class InvalidAuthenticationTypeExecption(exceptions.NeutronException):
message = (_('Invalid authentication type: %%(auth_type)s, '
'valid types are: %(valid_auth_types)s') %
{'valid_auth_types': ', '.join(VALID_AUTH_TYPES)})
class KeepalivedVipAddress(object):
"""A virtual address entry of a keepalived configuration."""
def __init__(self, ip_address, interface_name):
self.ip_address = ip_address
self.interface_name = interface_name
def build_config(self):
return '%s dev %s' % (self.ip_address, self.interface_name)
class KeepalivedVirtualRoute(object):
"""A virtual route entry of a keepalived configuration."""
def __init__(self, destination, nexthop, interface_name=None):
self.destination = destination
self.nexthop = nexthop
self.interface_name = interface_name
def build_config(self):
output = '%s via %s' % (self.destination, self.nexthop)
if self.interface_name:
output += ' dev %s' % self.interface_name
return output
class KeepalivedGroup(object):
"""Group section of a keepalived configuration."""
def __init__(self, ha_vr_id):
self.ha_vr_id = ha_vr_id
self.name = 'VG_%s' % ha_vr_id
self.instance_names = set()
self.notifiers = {}
def add_instance(self, instance):
self.instance_names.add(instance.name)
def set_notify(self, state, path):
if state not in VALID_NOTIFY_STATES:
raise InvalidNotifyStateException(state=state)
self.notifiers[state] = path
def build_config(self):
return itertools.chain(['vrrp_sync_group %s {' % self.name,
' group {'],
(' %s' % i for i in self.instance_names),
[' }'],
(' notify_%s "%s"' % (state, path)
for state, path in self.notifiers.items()),
['}'])
class KeepalivedInstance(object):
"""Instance section of a keepalived configuration."""
def __init__(self, state, interface, vrouter_id,
priority=HA_DEFAULT_PRIORITY, advert_int=None,
mcast_src_ip=None, nopreempt=False):
self.name = 'VR_%s' % vrouter_id
if state not in VALID_STATES:
raise InvalidInstanceStateException(state=state)
self.state = state
self.interface = interface
self.vrouter_id = vrouter_id
self.priority = priority
self.nopreempt = nopreempt
self.advert_int = advert_int
self.mcast_src_ip = mcast_src_ip
self.track_interfaces = []
self.vips = []
self.virtual_routes = []
self.authentication = tuple()
def set_authentication(self, auth_type, password):
if auth_type not in VALID_AUTH_TYPES:
raise InvalidAuthenticationTypeExecption(auth_type=auth_type)
self.authentication = (auth_type, password)
def remove_vips_vroutes_by_interface(self, interface_name):
self.vips = [vip for vip in self.vips
if vip.interface_name != interface_name]
self.virtual_routes = [vroute for vroute in self.virtual_routes
if vroute.interface_name != interface_name]
def remove_vip_by_ip_address(self, ip_address):
self.vips = [vip for vip in self.vips
if vip.ip_address != ip_address]
def _build_track_interface_config(self):
return itertools.chain(
[' track_interface {'],
(' %s' % i for i in self.track_interfaces),
[' }'])
def _build_vips_config(self):
vips_sorted = sorted(self.vips, key=lambda vip: vip.ip_address)
first_address = vips_sorted.pop(0)
vips_result = [' virtual_ipaddress {',
' %s' % first_address.build_config(),
' }']
if vips_sorted:
vips_result.extend(
itertools.chain([' virtual_ipaddress_excluded {'],
(' %s' % vip.build_config()
for vip in vips_sorted),
[' }']))
return vips_result
def _build_virtual_routes_config(self):
return itertools.chain([' virtual_routes {'],
(' %s' % route.build_config()
for route in self.virtual_routes),
[' }'])
def build_config(self):
config = ['vrrp_instance %s {' % self.name,
' state %s' % self.state,
' interface %s' % self.interface,
' virtual_router_id %s' % self.vrouter_id,
' priority %s' % self.priority]
if self.nopreempt:
config.append(' nopreempt')
if self.advert_int:
config.append(' advert_int %s' % self.advert_int)
if self.authentication:
auth_type, password = self.authentication
authentication = [' authentication {',
' auth_type %s' % auth_type,
' auth_pass %s' % password,
' }']
config.extend(authentication)
if self.mcast_src_ip:
config.append(' mcast_src_ip %s' % self.mcast_src_ip)
if self.track_interfaces:
config.extend(self._build_track_interface_config())
if self.vips:
config.extend(self._build_vips_config())
if self.virtual_routes:
config.extend(self._build_virtual_routes_config())
config.append('}')
return config
class KeepalivedConf(object):
"""A keepalived configuration."""
def __init__(self):
self.reset()
def reset(self):
self.groups = {}
self.instances = {}
def add_group(self, group):
self.groups[group.ha_vr_id] = group
def get_group(self, ha_vr_id):
return self.groups.get(ha_vr_id)
def add_instance(self, instance):
self.instances[instance.vrouter_id] = instance
def get_instance(self, vrouter_id):
return self.instances.get(vrouter_id)
def build_config(self):
config = []
for group in self.groups.values():
config.extend(group.build_config())
for instance in self.instances.values():
config.extend(instance.build_config())
return config
def get_config_str(self):
"""Generates and returns the keepalived configuration.
:return: Keepalived configuration string.
"""
return '\n'.join(self.build_config())
class KeepalivedNotifierMixin(object):
def _get_notifier_path(self, state):
return self._get_full_config_file_path('notify_%s.sh' % state)
def _write_notify_script(self, state, script):
name = self._get_notifier_path(state)
utils.replace_file(name, script)
st = os.stat(name)
os.chmod(name, st.st_mode | stat.S_IEXEC)
return name
def _prepend_shebang(self, script):
return '#!/usr/bin/env bash\n%s' % script
def _append_state(self, script, state):
state_path = self._get_full_config_file_path('state')
return '%s\necho -n %s > %s' % (script, state, state_path)
def add_notifier(self, script, state, ha_vr_id):
"""Add a master, backup or fault notifier.
These notifiers are executed when keepalived invokes a state
transition. Write a notifier to disk and add it to the
configuration.
"""
script_with_prefix = self._prepend_shebang(' '.join(script))
full_script = self._append_state(script_with_prefix, state)
self._write_notify_script(state, full_script)
group = self.config.get_group(ha_vr_id)
group.set_notify(state, self._get_notifier_path(state))
def get_conf_dir(self):
confs_dir = os.path.abspath(os.path.normpath(self.conf_path))
conf_dir = os.path.join(confs_dir, self.resource_id)
return conf_dir
def _get_full_config_file_path(self, filename, ensure_conf_dir=True):
conf_dir = self.get_conf_dir()
if ensure_conf_dir and not os.path.isdir(conf_dir):
os.makedirs(conf_dir, 0o755)
return os.path.join(conf_dir, filename)
class KeepalivedManager(KeepalivedNotifierMixin):
"""Wrapper for keepalived.
This wrapper permits to write keepalived config files, to start/restart
keepalived process.
"""
def __init__(self, resource_id, config, conf_path='/tmp',
namespace=None, root_helper=None):
self.resource_id = resource_id
self.config = config
self.namespace = namespace
self.root_helper = root_helper
self.conf_path = conf_path
self.conf = cfg.CONF
self.process = None
self.spawned = False
def _output_config_file(self):
config_str = self.config.get_config_str()
config_path = self._get_full_config_file_path('keepalived.conf')
utils.replace_file(config_path, config_str)
return config_path
def spawn(self):
config_path = self._output_config_file()
self.process = external_process.ProcessManager(
self.conf,
self.resource_id,
self.root_helper,
self.namespace,
pids_path=self.conf_path)
def callback(pid_file):
cmd = ['keepalived', '-P',
'-f', config_path,
'-p', pid_file,
'-r', '%s-vrrp' % pid_file]
return cmd
self.process.enable(callback)
self.spawned = True
LOG.debug('Keepalived spawned with config %s', config_path)
def spawn_or_restart(self):
if self.process:
self.restart()
else:
self.spawn()
def restart(self):
if self.process.active:
self._output_config_file()
self.process.reload_cfg()
else:
LOG.warn(_LW('A previous instance of keepalived seems to be dead, '
'unable to restart it, a new instance will be '
'spawned'))
self.process.disable()
self.spawn()
def disable(self):
if self.process:
self.process.disable(sig='15')
self.spawned = False
def revive(self):
if self.spawned and not self.process.active:
self.restart()

View File

@ -0,0 +1,60 @@
# Copyright (c) 2014 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from neutron.agent.common import config
from neutron.agent.linux import external_process
from neutron.agent.linux import keepalived
from neutron.openstack.common import log as logging
from neutron.tests.functional import base as functional_base
from neutron.tests.unit.agent.linux import test_keepalived
LOG = logging.getLogger(__name__)
class KeepalivedManagerTestCase(functional_base.BaseSudoTestCase,
test_keepalived.KeepalivedConfBaseMixin):
def setUp(self):
super(KeepalivedManagerTestCase, self).setUp()
self.check_sudo_enabled()
self._configure()
def _configure(self):
cfg.CONF.set_override('debug', True)
config.setup_logging(cfg.CONF)
config.register_root_helper(cfg.CONF)
cfg.CONF.set_override('root_helper', self.root_helper, group='AGENT')
def test_keepalived_spawn(self):
expected_config = self._get_config()
manager = keepalived.KeepalivedManager('router1', expected_config,
conf_path=cfg.CONF.state_path,
root_helper=self.root_helper)
self.addCleanup(manager.disable)
manager.spawn()
process = external_process.ProcessManager(
cfg.CONF,
'router1',
self.root_helper,
namespace=None,
pids_path=cfg.CONF.state_path)
self.assertTrue(process.active)
config_path = manager._get_full_config_file_path('keepalived.conf')
with open(config_path, 'r') as config_file:
config_contents = config_file.read()
self.assertEqual(expected_config.get_config_str(), config_contents)

View File

@ -0,0 +1,241 @@
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from neutron.agent.linux import keepalived
from neutron.tests import base
# Keepalived user guide:
# http://www.keepalived.org/pdf/UserGuide.pdf
class KeepalivedConfBaseMixin(object):
def _get_config(self):
config = keepalived.KeepalivedConf()
group1 = keepalived.KeepalivedGroup(1)
group2 = keepalived.KeepalivedGroup(2)
group1.set_notify('master', '/tmp/script.sh')
instance1 = keepalived.KeepalivedInstance('MASTER', 'eth0', 1,
advert_int=5)
instance1.set_authentication('AH', 'pass123')
instance1.track_interfaces.append("eth0")
vip_address1 = keepalived.KeepalivedVipAddress('192.168.1.0/24',
'eth1')
vip_address2 = keepalived.KeepalivedVipAddress('192.168.2.0/24',
'eth2')
vip_address3 = keepalived.KeepalivedVipAddress('192.168.3.0/24',
'eth2')
vip_address_ex = keepalived.KeepalivedVipAddress('192.168.55.0/24',
'eth10')
instance1.vips.append(vip_address1)
instance1.vips.append(vip_address2)
instance1.vips.append(vip_address3)
instance1.vips.append(vip_address_ex)
virtual_route = keepalived.KeepalivedVirtualRoute("0.0.0.0/0",
"192.168.1.1",
"eth1")
instance1.virtual_routes.append(virtual_route)
group1.add_instance(instance1)
instance2 = keepalived.KeepalivedInstance('MASTER', 'eth4', 2,
mcast_src_ip='224.0.0.1')
instance2.track_interfaces.append("eth4")
vip_address1 = keepalived.KeepalivedVipAddress('192.168.3.0/24',
'eth6')
instance2.vips.append(vip_address1)
instance2.vips.append(vip_address2)
instance2.vips.append(vip_address_ex)
group2.add_instance(instance2)
config.add_group(group1)
config.add_instance(instance1)
config.add_group(group2)
config.add_instance(instance2)
return config
class KeepalivedConfTestCase(base.BaseTestCase,
KeepalivedConfBaseMixin):
expected = """vrrp_sync_group VG_1 {
group {
VR_1
}
notify_master "/tmp/script.sh"
}
vrrp_sync_group VG_2 {
group {
VR_2
}
}
vrrp_instance VR_1 {
state MASTER
interface eth0
virtual_router_id 1
priority 50
advert_int 5
authentication {
auth_type AH
auth_pass pass123
}
track_interface {
eth0
}
virtual_ipaddress {
192.168.1.0/24 dev eth1
}
virtual_ipaddress_excluded {
192.168.2.0/24 dev eth2
192.168.3.0/24 dev eth2
192.168.55.0/24 dev eth10
}
virtual_routes {
0.0.0.0/0 via 192.168.1.1 dev eth1
}
}
vrrp_instance VR_2 {
state MASTER
interface eth4
virtual_router_id 2
priority 50
mcast_src_ip 224.0.0.1
track_interface {
eth4
}
virtual_ipaddress {
192.168.2.0/24 dev eth2
}
virtual_ipaddress_excluded {
192.168.3.0/24 dev eth6
192.168.55.0/24 dev eth10
}
}"""
def test_config_generation(self):
config = self._get_config()
self.assertEqual(self.expected, config.get_config_str())
def test_config_with_reset(self):
config = self._get_config()
self.assertEqual(self.expected, config.get_config_str())
config.reset()
self.assertEqual('', config.get_config_str())
class KeepalivedStateExceptionTestCase(base.BaseTestCase):
def test_state_exception(self):
group = keepalived.KeepalivedGroup('group2')
invalid_notify_state = 'a seal walks'
self.assertRaises(keepalived.InvalidNotifyStateException,
group.set_notify,
invalid_notify_state, '/tmp/script.sh')
invalid_vrrp_state = 'into a club'
self.assertRaises(keepalived.InvalidInstanceStateException,
keepalived.KeepalivedInstance,
invalid_vrrp_state, 'eth0', 33)
invalid_auth_type = '[hip, hip]'
instance = keepalived.KeepalivedInstance('MASTER', 'eth0', 1)
self.assertRaises(keepalived.InvalidAuthenticationTypeExecption,
instance.set_authentication,
invalid_auth_type, 'some_password')
class KeepalivedInstanceTestCase(base.BaseTestCase,
KeepalivedConfBaseMixin):
def test_remove_adresses_by_interface(self):
config = self._get_config()
instance = config.get_instance(1)
instance.remove_vips_vroutes_by_interface('eth2')
instance.remove_vips_vroutes_by_interface('eth10')
expected = """vrrp_sync_group VG_1 {
group {
VR_1
}
notify_master "/tmp/script.sh"
}
vrrp_sync_group VG_2 {
group {
VR_2
}
}
vrrp_instance VR_1 {
state MASTER
interface eth0
virtual_router_id 1
priority 50
advert_int 5
authentication {
auth_type AH
auth_pass pass123
}
track_interface {
eth0
}
virtual_ipaddress {
192.168.1.0/24 dev eth1
}
virtual_routes {
0.0.0.0/0 via 192.168.1.1 dev eth1
}
}
vrrp_instance VR_2 {
state MASTER
interface eth4
virtual_router_id 2
priority 50
mcast_src_ip 224.0.0.1
track_interface {
eth4
}
virtual_ipaddress {
192.168.2.0/24 dev eth2
}
virtual_ipaddress_excluded {
192.168.3.0/24 dev eth6
192.168.55.0/24 dev eth10
}
}"""
self.assertEqual(expected, config.get_config_str())
class KeepalivedVirtualRouteTestCase(base.BaseTestCase):
def test_virtual_route_with_dev(self):
route = keepalived.KeepalivedVirtualRoute('0.0.0.0/0', '1.2.3.4',
'eth0')
self.assertEqual('0.0.0.0/0 via 1.2.3.4 dev eth0',
route.build_config())
def test_virtual_route_without_dev(self):
route = keepalived.KeepalivedVirtualRoute('50.0.0.0/8', '1.2.3.4')
self.assertEqual('50.0.0.0/8 via 1.2.3.4', route.build_config())