openstack/trove
Code Issues Proposed changes

Changed system commands depends on OS version

Browse Source 
fixes bug #1214120

Change-Id: I3357dd04911a92a4d69fb8d3469f093b0c6b0737
This commit is contained in:
Andrey Shestakov 2013-08-20 19:34:05 +03:00
parent 1467437f83
commit 58534eda0d
17 changed files with 102 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -22,4 +22,4 @@ Jinja2
pexpect
-f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3
oslo.config>=1.2.0a3
mysql-python

1
setup.cfg
View File

@ -27,6 +27,7 @@ scripts =
bin/trove-mgmt-taskmanager
bin/trove-server
bin/trove-taskmanager
bin/trove-guestagent
[global]
setup-hooks =

4
trove/common/template.py
View File

@ -23,9 +23,6 @@ ENV = jinja2.Environment(loader=jinja2.ChoiceLoader([
class SingleInstanceConfigTemplate(object):
""" This class selects a single configuration file by database type for
rendering on the guest """
_config_paths = {'mysql': '/etc/mysql/my.cnf',
'percona': '/etc/mysql/my.cnf'}
def __init__(self, service_type, flavor_dict):
""" Constructor
@ -35,7 +32,6 @@ class SingleInstanceConfigTemplate(object):
:type flavor_dict: dict.
"""
self.config_location = self._config_paths[service_type]
self.flavor_dict = flavor_dict
template_filename = "%s.config.template" % service_type
self.template = ENV.get_template(template_filename)

8
trove/guestagent/api.py
View File

@ -211,7 +211,7 @@ class API(proxy.RpcProxy):
def prepare(self, memory_mb, databases, users,
device_path='/dev/vdb', mount_point='/mnt/volume',
backup_id=None, config_location=None, config_contents=None):
backup_id=None, config_contents=None):
"""Make an asynchronous call to prepare the guest
as a database container optionally includes a backup id for restores
"""
@ -219,20 +219,18 @@ class API(proxy.RpcProxy):
self._cast_with_consumer(
"prepare", databases=databases, memory_mb=memory_mb,
users=users, device_path=device_path, mount_point=mount_point,
backup_id=backup_id, config_location=config_location,
config_contents=config_contents)
backup_id=backup_id, config_contents=config_contents)
def restart(self):
"""Restart the MySQL server."""
LOG.debug(_("Sending the call to restart MySQL on the Guest."))
self._call("restart", AGENT_HIGH_TIMEOUT)
def start_db_with_conf_changes(self, config_location, config_contents):
def start_db_with_conf_changes(self, config_contents):
"""Start the MySQL server."""
LOG.debug(_("Sending the call to start MySQL on the Guest with "
"a timeout of %s.") % AGENT_HIGH_TIMEOUT)
self._call("start_db_with_conf_changes", AGENT_HIGH_TIMEOUT,
config_location=config_location,
config_contents=config_contents)
def reset_configuration(self, configuration):

10
trove/guestagent/manager/mysql.py
View File

@ -85,8 +85,7 @@ class Manager(periodic_task.PeriodicTasks):
LOG.info(_("Restored database successfully"))
def prepare(self, context, databases, memory_mb, users, device_path=None,
mount_point=None, backup_id=None, config_location=None,
config_contents=None):
mount_point=None, backup_id=None, config_contents=None):
"""Makes ready DBAAS on a Guest container."""
MySqlAppStatus.get().begin_mysql_install()
# status end_mysql_install set with secure()
@ -113,7 +112,7 @@ class Manager(periodic_task.PeriodicTasks):
if backup_id:
self._perform_restore(backup_id, context, CONF.mount_point, app)
LOG.info(_("Securing mysql now."))
app.secure(config_location, config_contents)
app.secure(config_contents)
enable_root_on_restore = (backup_id and MySqlAdmin().is_root_enabled())
if enable_root_on_restore:
MySqlAdmin().report_root_enabled(context)
@ -132,10 +131,9 @@ class Manager(periodic_task.PeriodicTasks):
app = MySqlApp(MySqlAppStatus.get())
app.restart()
def start_db_with_conf_changes(self, context, config_location,
config_contents):
def start_db_with_conf_changes(self, context, config_contents):
app = MySqlApp(MySqlAppStatus.get())
app.start_db_with_conf_changes(config_location, config_contents)
app.start_db_with_conf_changes(config_contents)
def stop_db(self, context, do_not_start_on_reboot=False):
app = MySqlApp(MySqlAppStatus.get())

37
trove/guestagent/manager/mysql_service.py
View File

@ -14,6 +14,7 @@ from trove.common import utils as utils
from trove.common import exception
from trove.guestagent import query
from trove.guestagent.db import models
from trove.guestagent import system
from trove.guestagent import pkg
from trove.instance import models as rd_models
from trove.openstack.common import log as logging
@ -51,7 +52,7 @@ def get_auth_password():
"sudo",
"awk",
"/password\\t=/{print $3; exit}",
"/etc/mysql/my.cnf")
system.MYSQL_CONFIG)
if err:
LOG.error(err)
raise RuntimeError("Problem reading my.cnf! : %s" % err)
@ -77,7 +78,7 @@ def get_engine():
def load_mysqld_options():
try:
out, err = utils.execute("/usr/sbin/mysqld", "--print-defaults",
out, err = utils.execute(system.MYSQL_BIN, "--print-defaults",
run_as_root=True, root_helper="sudo")
arglist = re.split("\n", out)[1].split()
args = {}
@ -650,7 +651,7 @@ class MySqlApp(object):
def complete_install_or_restart(self):
self.status.end_install_or_restart()
def secure(self, config_location, config_contents):
def secure(self, config_contents):
LOG.info(_("Generating admin password..."))
admin_password = generate_random_password()
@ -661,7 +662,7 @@ class MySqlApp(object):
self._create_admin_user(client, admin_password)
self.stop_db()
self._write_mycnf(admin_password, config_location, config_contents)
self._write_mycnf(admin_password, config_contents)
self.start_mysql()
LOG.info(_("Dbaas secure complete."))
@ -679,6 +680,7 @@ class MySqlApp(object):
"""Install mysql server. The current version is 5.5"""
LOG.debug(_("Installing mysql server"))
packager.pkg_install(self.MYSQL_PACKAGE_VERSION, self.TIME_OUT)
self.start_mysql()
LOG.debug(_("Finished installing mysql server"))
#TODO(rnirmal): Add checks to make sure the package got installed
@ -697,9 +699,7 @@ class MySqlApp(object):
command = "sudo sed -i '/^manual$/d' %(conf)s"
command = command % locals()
else:
command = "sudo update-rc.d mysql enable"
if pkg.OS == pkg.REDHAT:
command = "sudo chkconfig mysql on"
command = system.MYSQL_CMD_ENABLE
utils.execute_with_timeout(command, shell=True)
def _disable_mysql_on_boot(self):
@ -717,16 +717,14 @@ class MySqlApp(object):
command = '''sudo sh -c "echo manual >> %(conf)s"'''
command = command % locals()
else:
command = "sudo update-rc.d mysql disable"
if pkg.OS == pkg.REDHAT:
command = "sudo chkconfig mysql off"
command = system.MYSQL_CMD_DISABLE
utils.execute_with_timeout(command, shell=True)
def stop_db(self, update_db=False, do_not_start_on_reboot=False):
LOG.info(_("Stopping mysql..."))
if do_not_start_on_reboot:
self._disable_mysql_on_boot()
utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "stop")
utils.execute_with_timeout(system.MYSQL_CMD_STOP, shell=True)
if not self.status.wait_for_real_status_to_change_to(
rd_models.ServiceStatuses.SHUTDOWN,
self.state_change_wait_time, update_db):
@ -798,7 +796,7 @@ class MySqlApp(object):
if "No such file or directory" not in str(pe):
raise
def _write_mycnf(self, admin_password, config_location, config_contents):
def _write_mycnf(self, admin_password, config_contents):
"""
Install the set of mysql my.cnf templates.
Update the os_admin user and password to the my.cnf
@ -811,13 +809,13 @@ class MySqlApp(object):
with open(TMP_MYCNF, 'w') as t:
t.write(config_contents)
utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
config_location)
system.MYSQL_CONFIG)
self._write_temp_mycnf_with_admin_account(config_location,
self._write_temp_mycnf_with_admin_account(system.MYSQL_CONFIG,
TMP_MYCNF,
admin_password)
utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
config_location)
system.MYSQL_CONFIG)
self.wipe_ib_logfiles()
@ -830,7 +828,7 @@ class MySqlApp(object):
self._enable_mysql_on_boot()
try:
utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start")
utils.execute_with_timeout(system.MYSQL_CMD_START, shell=True)
except exception.ProcessExecutionError:
# it seems mysql (percona, at least) might come back with [Fail]
# but actually come up ok. we're looking into the timing issue on
@ -853,7 +851,7 @@ class MySqlApp(object):
self.status.end_install_or_restart()
raise RuntimeError("Could not start MySQL!")
def start_db_with_conf_changes(self, config_location, config_contents):
def start_db_with_conf_changes(self, config_contents):
LOG.info(_("Starting mysql with conf changes..."))
LOG.info(_("inside the guest - self.status.is_mysql_running(%s)...")
% self.status.is_mysql_running)
@ -862,14 +860,13 @@ class MySqlApp(object):
"MySQL state == %s!") % self.status)
raise RuntimeError("MySQL not stopped.")
LOG.info(_("Initiating config."))
self._write_mycnf(None, config_location, config_contents)
self._write_mycnf(None, config_contents)
self.start_mysql(True)
def reset_configuration(self, configuration):
config_location = configuration['config_location']
config_contents = configuration['config_contents']
LOG.info(_("Resetting configuration"))
self._write_mycnf(None, config_location, config_contents)
self._write_mycnf(None, config_contents)
def is_installed(self):
#(cp16net) could raise an exception, does it need to be handled here?

18
trove/guestagent/pkg.py
View File

@ -27,6 +27,7 @@ from trove.common import exception
from trove.common import utils
from trove.common.exception import ProcessExecutionError
from trove.openstack.common import log as logging
from trove.guestagent import system
from trove.openstack.common.gettextutils import _
@ -34,13 +35,6 @@ LOG = logging.getLogger(__name__)
OK = 0
RUN_DPKG_FIRST = 1
REINSTALL_FIRST = 2
REDHAT = 'redhat'
DEBIAN = 'debian'
# The default is debian
OS = DEBIAN
if os.path.isfile("/etc/redhat-release"):
OS = REDHAT
class PkgAdminLockError(exception.TroveError):
@ -82,14 +76,14 @@ class BasePackagerMixin:
child.delayafterterminate = 1
child.close(force=True)
def pexpect_wait_and_close_proc(self, child, time_out=-1):
child.expect(pexpect.EOF, timeout=time_out)
def pexpect_wait_and_close_proc(self, child):
child.expect(pexpect.EOF)
child.close()
def pexpect_run(self, cmd, output_expects, time_out):
child = pexpect.spawn(cmd)
child = pexpect.spawn(cmd, timeout=time_out)
try:
i = child.expect(output_expects, timeout=time_out)
i = child.expect(output_expects)
self.pexpect_wait_and_close_proc(child)
except pexpect.TIMEOUT:
self.pexpect_kill_proc(child)
@ -324,7 +318,7 @@ class DebianPackagerMixin(BasePackagerMixin):
class BasePackage(type):
def __new__(meta, name, bases, dct):
if OS == REDHAT:
if system.OS == system.REDHAT:
bases += (RedhatPackagerMixin, )
else:
# The default is debian

51
trove/guestagent/system.py Normal file
View File

@ -0,0 +1,51 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Determines operating system version and os depended commands.
"""
import os.path
from trove.common import cfg
CONF = cfg.CONF
REDHAT = 'redhat'
DEBIAN = 'debian'
# The default is debian
OS = DEBIAN
MYSQL_CONFIG = "/etc/mysql/my.cnf"
MYSQL_BIN = "/usr/sbin/mysqld"
MYSQL_CMD_ENABLE = "sudo update-rc.d mysql enable"
MYSQL_CMD_DISABLE = "sudo update-rc.d mysql disable"
MYSQL_CMD_START = "sudo service mysql start || /bin/true"
MYSQL_CMD_STOP = "sudo service mysql stop || /bin/true"
if os.path.isfile("/etc/redhat-release"):
OS = REDHAT
MYSQL_CONFIG = "/etc/my.cnf"
if CONF.service_type == 'percona':
MYSQL_CMD_ENABLE = "sudo chkconfig mysql on"
MYSQL_CMD_DISABLE = "sudo chkconfig mysql off"
MYSQL_CMD_START = "sudo service mysql start"
MYSQL_CMD_STOP = "sudo service mysql stop"
else:
MYSQL_BIN = "/usr/libexec/mysqld"
MYSQL_CMD_ENABLE = "sudo chkconfig mysqld on"
MYSQL_CMD_DISABLE = "sudo chkconfig mysqld off"
MYSQL_CMD_START = "sudo service mysqld start"
MYSQL_CMD_STOP = "sudo service mysqld stop"

10
trove/taskmanager/models.py
View File

@ -154,7 +154,6 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
if server:
self._guest_prepare(server, flavor['ram'], volume_info,
databases, users, backup_id,
config.config_location,
config.config_contents)
if not self.db_info.task_status.is_error:
@ -353,14 +352,13 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
def _guest_prepare(self, server, flavor_ram, volume_info,
databases, users, backup_id=None,
config_location=None, config_contents=None):
config_contents=None):
LOG.info("Entering guest_prepare.")
# Now wait for the response from the create to do additional work
self.guest.prepare(flavor_ram, databases, users,
device_path=volume_info['device_path'],
mount_point=volume_info['mount_point'],
backup_id=backup_id,
config_location=config_location,
config_contents=config_contents)
def _create_dns_entry(self):
@ -776,8 +774,7 @@ class ResizeAction(ResizeActionBase):
try:
config = self._render_config(self.instance.service_type,
self.old_flavor)
config = {'config_location': config.config_location,
'config_contents': config.config_contents}
config = {'config_contents': config.config_contents}
self.instance.guest.reset_configuration(config)
except GuestTimeout as gt:
LOG.exception("Error sending reset_configuration call.")
@ -798,8 +795,7 @@ class ResizeAction(ResizeActionBase):
def _start_mysql(self):
config = self._render_config(self.instance.service_type,
self.new_flavor)
self.instance.guest.start_db_with_conf_changes(config.config_location,
config.config_contents)
self.instance.guest.start_db_with_conf_changes(config.config_contents)
class MigrateAction(ResizeActionBase):

5
trove/templates/mysql.config.template
View File

@ -1,14 +1,11 @@
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
socket = /var/run/mysqld/mysqld.sock
nice = 0
[mysqld]
user = mysql
socket = /var/run/mysqld/mysqld.sock
port = 3306
basedir = /usr
datadir = /var/lib/mysql
@ -24,7 +21,6 @@ myisam-recover = BACKUP
query_cache_type = 1
query_cache_limit = 1M
query_cache_size = {{ 8 * flavor['ram']//512 }}M
log_error = /var/log/mysql/mysqld.log
innodb_data_file_path = ibdata1:10M:autoextend
innodb_buffer_pool_size = {{ 150 * flavor['ram']//512 }}M
innodb_file_per_table = 1
@ -55,4 +51,3 @@ max_allowed_packet = 16M
[isamchk]
key_buffer = 16M
!includedir /etc/mysql/conf.d/

5
trove/templates/percona.config.template
View File

@ -1,14 +1,11 @@
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld_safe]
socket = /var/run/mysqld/mysqld.sock
nice = 0
[mysqld]
user = mysql
socket = /var/run/mysqld/mysqld.sock
port = 3306
basedir = /usr
datadir = /var/lib/mysql
@ -24,7 +21,6 @@ myisam-recover = BACKUP
query_cache_type = 1
query_cache_limit = 1M
query_cache_size = {{ 8 * flavor['ram']//512 }}M
log_error = /var/log/mysql/mysqld.log
innodb_data_file_path = ibdata1:10M:autoextend
innodb_buffer_pool_size = {{ 150 * flavor['ram']//512 }}M
innodb_file_per_table = 1
@ -55,4 +51,3 @@ max_allowed_packet = 16M
[isamchk]
key_buffer = 16M
!includedir /etc/mysql/conf.d/

3
trove/tests/api/instances_resize.py
View File

@ -117,8 +117,7 @@ class ResizeTests(ResizeTestBase):
def _start_mysql(self):
config = template.SingleInstanceConfigTemplate(
"mysql", NEW_FLAVOR.__dict__)
self.instance.guest.start_db_with_conf_changes(config.config_location,
config.render())
self.instance.guest.start_db_with_conf_changes(config.render())
def test_guest_wont_stop_mysql(self):
self.guest.stop_db(do_not_start_on_reboot=True)\

5
trove/tests/fakes/guestagent.py
View File

@ -208,8 +208,7 @@ class FakeGuest(object):
return self.users.get((username, hostname), None)
def prepare(self, memory_mb, databases, users, device_path=None,
mount_point=None, backup_id=None, config_location=None,
config_contents=None):
mount_point=None, backup_id=None, config_contents=None):
from trove.instance.models import DBInstance
from trove.instance.models import InstanceServiceStatus
from trove.instance.models import ServiceStatuses
@ -253,7 +252,7 @@ class FakeGuest(object):
# There's nothing to do here, since there is no config to update.
pass
def start_db_with_conf_changes(self, config_location, config_contents):
def start_db_with_conf_changes(self, config_contents):
time.sleep(2)
self._set_status('RUNNING')

2
trove/tests/unittests/common/test_template.py
View File

@ -48,9 +48,7 @@ class TemplateTest(testtools.TestCase):
self.validate_template(rendered, "query_cache_size", self.flavor_dict)
def test_single_instance_config_rendering(self):
location = "/etc/mysql/my.cnf"
config = template.SingleInstanceConfigTemplate('mysql',
self.flavor_dict)
self.assertEqual(location, config.config_location)
self.validate_template(config.render(), "query_cache_size",
self.flavor_dict)

12
trove/tests/unittests/guestagent/test_api.py
View File

@ -177,9 +177,9 @@ class ApiTest(testtools.TestCase):
def test_start_db_with_conf_changes(self):
exp_msg = RpcMsgMatcher('start_db_with_conf_changes',
'config_location', 'config_contents')
'config_contents')
self._mock_rpc_call(exp_msg)
self.api.start_db_with_conf_changes(None, None)
self.api.start_db_with_conf_changes(None)
self._verify_rpc_call(exp_msg)
def test_stop_db(self):
@ -219,12 +219,12 @@ class ApiTest(testtools.TestCase):
when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None)
exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users',
'device_path', 'mount_point', 'backup_id',
'config_location', 'config_contents')
'config_contents')
when(rpc).cast(any(), any(), exp_msg).thenReturn(None)
self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt',
'bkup-1232', 'loc', 'cont')
'bkup-1232', 'cont')
self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg)
@ -234,11 +234,11 @@ class ApiTest(testtools.TestCase):
when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None)
exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users',
'device_path', 'mount_point', 'backup_id',
'config_location', 'config_contents')
'config_contents')
when(rpc).cast(any(), any(), exp_msg).thenReturn(None)
self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt',
'backup_id_123', 'loc', 'cont')
'backup_id_123', 'cont')
self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg)

14
trove/tests/unittests/guestagent/test_dbaas.py
View File

@ -60,7 +60,6 @@ FAKE_DB_2 = {"_name": "testDB2", "_character_set": "latin2",
"_collate": "latin2_general_ci"}
FAKE_USER = [{"_name": "random", "_password": "guesswhat",
"_databases": [FAKE_DB]}]
MYCNF = '/etc/mysql/my.cnf'
class FakeAppStatus(MySqlAppStatus):
@ -595,7 +594,7 @@ class MySqlAppTest(testtools.TestCase):
self.mysql_starts_successfully()
self.appStatus.status = ServiceStatuses.SHUTDOWN
self.mySqlApp.start_db_with_conf_changes(Mock(), Mock())
self.mySqlApp.start_db_with_conf_changes(Mock())
self.assertTrue(self.mySqlApp._write_mycnf.called)
self.assertTrue(self.mySqlApp.start_mysql.called)
@ -610,7 +609,7 @@ class MySqlAppTest(testtools.TestCase):
self.appStatus.status = ServiceStatuses.RUNNING
self.assertRaises(RuntimeError,
self.mySqlApp.start_db_with_conf_changes,
Mock(), Mock())
Mock())
class MySqlAppInstallTest(MySqlAppTest):
@ -642,7 +641,7 @@ class MySqlAppInstallTest(MySqlAppTest):
self.mysql_starts_successfully()
sqlalchemy.create_engine = Mock()
self.mySqlApp.secure(MYCNF, 'contents')
self.mySqlApp.secure('contents')
self.assertTrue(self.mySqlApp.stop_db.called)
self.assertTrue(self.mySqlApp._write_mycnf.called)
@ -674,8 +673,7 @@ class MySqlAppInstallTest(MySqlAppTest):
self.mysql_starts_successfully()
sqlalchemy.create_engine = Mock()
self.assertRaises(IOError,
self.mySqlApp.secure, "/etc/mycnf/my.cnf", "foo")
self.assertRaises(IOError, self.mySqlApp.secure, "foo")
self.assertTrue(self.mySqlApp.stop_db.called)
self.assertTrue(self.mySqlApp._write_mycnf.called)
@ -748,7 +746,7 @@ class MySqlAppMockTest(testtools.TestCase):
any(), any(), any()).thenReturn(True)
app = MySqlApp(mock_status)
self.assertRaises(TypeError, app.secure, MYCNF, None)
self.assertRaises(TypeError, app.secure, None)
verify(mock_conn, atleast=2).execute(any())
inorder.verify(mock_status).wait_for_real_status_to_change_to(
@ -770,7 +768,7 @@ class MySqlAppMockTest(testtools.TestCase):
any(), any(), any()).thenReturn(True)
app = MySqlApp(mock_status)
when(app)._write_mycnf(any(), any()).thenReturn(True)
app.secure(MYCNF, 'foo')
app.secure('foo')
verify(mock_conn, never).execute(TextClauseMatcher('root'))

2
trove/tests/unittests/guestagent/test_manager.py
View File

@ -182,7 +182,7 @@ class GuestAgentManagerTest(testtools.TestCase):
verify(backup).restore(self.context, backup_id, '/var/lib/mysql')
verify(dbaas.MySqlApp).install_if_needed()
# We dont need to make sure the exact contents are there
verify(dbaas.MySqlApp).secure(any(), any())
verify(dbaas.MySqlApp).secure(any())
verify(dbaas.MySqlAdmin, never).create_database()
verify(dbaas.MySqlAdmin, never).create_user()
times_report = 1 if is_root_enabled else 0