Implement Instance Upgrade

Implments Instance Upgrade functionality to support upgrading the
image of a Trove datastore instance from datastore_version to a
newer datastore_version of the same datastore.

This functionality builds on the Nova rebuild API to upgrade
the image of an instance with minimal downtime.

Includes datastore implementation of the upgrade functionality
for the Mysql based datastores.

Change-Id: Ie6e48d78ac07df52f686f359ca7fdadaae6ad064
Implements: blueprint image-upgrade
Depends-On: I6ec2ebb78019c014f87ba5d8cbfd284686c64f30

releasenotes/notes/instance-upgrade-7d464f85e025d729.yaml

@@ -0,0 +1,4 @@
+features:
+  - New instance upgrade API supports upgrading an instance of
+    a datastore to a new datastore version.  Includes implementation
+    for MySQL family of databases.

trove/common/apischema.py

@@ -370,6 +370,7 @@ instance = {
                     "replica_of": {},
                     "name": non_empty_string,
                     "configuration": configuration_id,
+                    "datastore_version": non_empty_string,
                 }
             }
         }

trove/common/notification.py

@@ -347,6 +347,10 @@ class DBaaSAPINotification(object):
     def server_type(self, server_type):
         self.payload['server_type'] = server_type
 
+    @property
+    def request_id(self):
+        return self.payload['request_id']
+
     def __init__(self, context, **kwargs):
         self.context = context
         self.needs_end_notification = True
@@ -753,3 +757,14 @@ class DBaaSConfigurationEdit(DBaaSAPINotification):
     @abc.abstractmethod
     def required_start_traits(self):
         return ['configuration_id']
+
+
+class DBaaSInstanceUpgrade(DBaaSAPINotification):
+
+    @abc.abstractmethod
+    def event_type(self):
+        return 'upgrade'
+
+    @abc.abstractmethod
+    def required_start_traits(self):
+        return ['instance_id', 'datastore_version_id']

trove/conductor/manager.py

@@ -149,4 +149,6 @@ class Manager(periodic_task.PeriodicTasks):
                         message, exception):
         notification = SerializableNotification.deserialize(
             context, serialized_notification)
+        LOG.error(_("Guest exception on request %(req)s:\n%(exc)s")
+                  % {'req': notification.request_id, 'exc': exception})
         notification.notify_exc_info(message, exception)

trove/datastore/models.py

@@ -336,6 +336,9 @@ class Datastore(object):
     def __init__(self, db_info):
         self.db_info = db_info
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @classmethod
     def load(cls, id_or_name):
         try:
@@ -387,6 +390,9 @@ class DatastoreVersion(object):
         self.db_info = db_info
         self._datastore_name = None
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @classmethod
     def load(cls, datastore, id_or_name):
         try:

trove/guestagent/api.py

@@ -267,6 +267,17 @@ class API(object):
                 server.stop()
                 server.wait()
 
+    def pre_upgrade(self):
+        """Prepare the guest for upgrade."""
+        LOG.debug("Sending the call to prepare the guest for upgrade.")
+        return self._call("pre_upgrade", AGENT_HIGH_TIMEOUT, self.version_cap)
+
+    def post_upgrade(self, upgrade_info):
+        """Recover the guest after upgrading the guest's image."""
+        LOG.debug("Recover the guest after upgrading the guest's image.")
+        self._call("post_upgrade", AGENT_HIGH_TIMEOUT, self.version_cap,
+                   upgrade_info=upgrade_info)
+
     def restart(self):
         """Restart the database server."""
         LOG.debug("Sending the call to restart the database process "

trove/guestagent/common/configuration.py

@@ -96,7 +96,7 @@ class ConfigurationManager(object):
         """Return the current value at a given key or 'default'.
         """
         if self._value_cache is None:
-            self._refresh_cache()
+            self.refresh_cache()
 
         return self._value_cache.get(key, default)
 
@@ -139,7 +139,7 @@ class ConfigurationManager(object):
                 self._base_config_path, FileMode.ADD_READ_ALL,
                 as_root=self._requires_root)
 
-            self._refresh_cache()
+            self.refresh_cache()
 
     def has_system_override(self, change_id):
         """Return whether a given 'system' change exists.
@@ -178,7 +178,7 @@ class ConfigurationManager(object):
                 group_name, change_id, self._codec.deserialize(options))
         else:
             self._override_strategy.apply(group_name, change_id, options)
-            self._refresh_cache()
+            self.refresh_cache()
 
     def remove_system_override(self, change_id=DEFAULT_CHANGE_ID):
         """Revert a 'system' configuration change.
@@ -192,9 +192,9 @@ class ConfigurationManager(object):
 
     def _remove_override(self, group_name, change_id):
         self._override_strategy.remove(group_name, change_id)
-        self._refresh_cache()
+        self.refresh_cache()
 
-    def _refresh_cache(self):
+    def refresh_cache(self):
         self._value_cache = self.parse_configuration()
 
 

trove/guestagent/datastore/manager.py

@@ -252,7 +252,7 @@ class Manager(periodic_task.PeriodicTasks):
         return True
 
     #################
-    # Prepare related
+    # Instance related
     #################
     def prepare(self, context, packages, databases, memory_mb, users,
                 device_path=None, mount_point=None, backup_info=None,
@@ -389,6 +389,18 @@ class Manager(periodic_task.PeriodicTasks):
         LOG.info(_('No post_prepare work has been defined.'))
         pass
 
+    def pre_upgrade(self, context):
+        """Prepares the guest for upgrade, returning a dict to be passed
+        to post_upgrade
+        """
+        return {}
+
+    def post_upgrade(self, context, upgrade_info):
+        """Recovers the guest after the image is upgraded using infomation
+        from the pre_upgrade step
+        """
+        pass
+
     #################
     # Service related
     #################
@@ -407,11 +419,12 @@ class Manager(periodic_task.PeriodicTasks):
         LOG.debug("Getting file system stats for '%s'" % mount_point)
         return dbaas.get_filesystem_volume_stats(mount_point)
 
-    def mount_volume(self, context, device_path=None, mount_point=None):
+    def mount_volume(self, context, device_path=None, mount_point=None,
+                     write_to_fstab=False):
         LOG.debug("Mounting the device %s at the mount point %s." %
                   (device_path, mount_point))
         device = volume.VolumeDevice(device_path)
-        device.mount(mount_point, write_to_fstab=False)
+        device.mount(mount_point, write_to_fstab=write_to_fstab)
 
     def unmount_volume(self, context, device_path=None, mount_point=None):
         LOG.debug("Unmounting the device %s from the mount point %s." %

trove/guestagent/datastore/mysql_common/manager.py

@@ -242,6 +242,56 @@ class MySqlManager(manager.Manager):
         if snapshot:
             self.attach_replica(context, snapshot, snapshot['config'])
 
+    def pre_upgrade(self, context):
+        app = self.mysql_app(self.mysql_app_status.get())
+        data_dir = app.get_data_dir()
+        mount_point, _data = os.path.split(data_dir)
+        save_dir = "%s/etc_mysql" % mount_point
+        save_etc_dir = "%s/etc" % mount_point
+        home_save = "%s/trove_user" % mount_point
+
+        app.status.begin_restart()
+        app.stop_db()
+
+        if operating_system.exists("/etc/my.cnf", as_root=True):
+            operating_system.create_directory(save_etc_dir, as_root=True)
+            operating_system.copy("/etc/my.cnf", save_etc_dir,
+                                  preserve=True, as_root=True)
+
+        operating_system.copy("/etc/mysql/.", save_dir,
+                              preserve=True, as_root=True)
+
+        operating_system.copy("%s/." % os.path.expanduser('~'), home_save,
+                              preserve=True, as_root=True)
+
+        self.unmount_volume(context, mount_point=data_dir)
+        return {
+            'mount_point': mount_point,
+            'save_dir': save_dir,
+            'save_etc_dir': save_etc_dir,
+            'home_save': home_save
+        }
+
+    def post_upgrade(self, context, upgrade_info):
+        app = self.mysql_app(self.mysql_app_status.get())
+        app.stop_db()
+        if 'device' in upgrade_info:
+            self.mount_volume(context, mount_point=upgrade_info['mount_point'],
+                              device_path=upgrade_info['device'],
+                              write_to_fstab=True)
+
+        if operating_system.exists(upgrade_info['save_etc_dir'],
+                                   is_directory=True, as_root=True):
+            operating_system.copy("%s/." % upgrade_info['save_etc_dir'],
+                                  "/etc", preserve=True, as_root=True)
+
+        operating_system.copy("%s/." % upgrade_info['save_dir'], "/etc/mysql",
+                              preserve=True, as_root=True)
+        operating_system.copy("%s/." % upgrade_info['home_save'],
+                              os.path.expanduser('~'),
+                              preserve=True, as_root=True)
+        app.start_mysql()
+
     def restart(self, context):
         app = self.mysql_app(self.mysql_app_status.get())
         app.restart()

trove/instance/models.py

@@ -17,6 +17,7 @@
 """Model classes that form the core of instances functionality."""
 from datetime import datetime
 from datetime import timedelta
+import os.path
 import re
 
 from novaclient import exceptions as nova_exceptions
@@ -98,6 +99,7 @@ class InstanceStatus(object):
     RESTART_REQUIRED = "RESTART_REQUIRED"
     PROMOTE = "PROMOTE"
     EJECT = "EJECT"
+    UPGRADE = "UPGRADE"
     DETACH = "DETACH"
 
 
@@ -129,7 +131,8 @@ def load_simple_instance_server_status(context, db_info):
 
 
 # Invalid states to contact the agent
-AGENT_INVALID_STATUSES = ["BUILD", "REBOOT", "RESIZE", "PROMOTE", "EJECT"]
+AGENT_INVALID_STATUSES = ["BUILD", "REBOOT", "RESIZE", "PROMOTE", "EJECT",
+                          "UPGRADE"]
 
 
 class SimpleInstance(object):
@@ -175,6 +178,9 @@ class SimpleInstance(object):
 
         self.slave_list = None
 
+    def __repr__(self, *args, **kwargs):
+        return "%s(%s)" % (self.name, self.id)
+
     @property
     def addresses(self):
         # TODO(tim.simpson): This code attaches two parts of the Nova server to
@@ -296,6 +302,8 @@ class SimpleInstance(object):
             return InstanceStatus.REBOOT
         if 'RESIZING' == action:
             return InstanceStatus.RESIZE
+        if 'UPGRADING' == action:
+            return InstanceStatus.UPGRADE
         if 'RESTART_REQUIRED' == action:
             return InstanceStatus.RESTART_REQUIRED
         if InstanceTasks.PROMOTING.action == action:
@@ -684,6 +692,32 @@ class BaseInstance(SimpleInstance):
             self._server_group_loaded = True
         return self._server_group
 
+    def get_injected_files(self, datastore_manager):
+        injected_config_location = CONF.get('injected_config_location')
+        guest_info = CONF.get('guest_info')
+
+        if ('/' in guest_info):
+            # Set guest_info_file to exactly guest_info from the conf file.
+            # This should be /etc/guest_info for pre-Kilo compatibility.
+            guest_info_file = guest_info
+        else:
+            guest_info_file = os.path.join(injected_config_location,
+                                           guest_info)
+
+        files = {guest_info_file: (
+            "[DEFAULT]\n"
+            "guest_id=%s\n"
+            "datastore_manager=%s\n"
+            "tenant_id=%s\n"
+            % (self.id, datastore_manager, self.tenant_id))}
+
+        if os.path.isfile(CONF.get('guest_config')):
+            with open(CONF.get('guest_config'), "r") as f:
+                files[os.path.join(injected_config_location,
+                                   "trove-guestagent.conf")] = f.read()
+
+        return files
+
 
 class FreshInstance(BaseInstance):
     @classmethod
@@ -1230,6 +1264,12 @@ class Instance(BuiltInstance):
             self.datastore_version, flavor, self.id)
         return dict(config.render_dict())
 
+    def upgrade(self, datastore_version):
+        self.update_db(datastore_version_id=datastore_version.id,
+                       task_status=InstanceTasks.UPGRADING)
+        task_api.API(self.context).upgrade(self.id,
+                                           datastore_version.id)
+
 
 def create_server_list_matcher(server_list):
     # Returns a method which finds a server from the given list.

trove/instance/service.py

@@ -312,15 +312,6 @@ class InstanceController(wsgi.Controller):
                 return configuration_id
 
     def _modify_instance(self, context, req, instance, **kwargs):
-        """Modifies the instance using the specified keyword arguments
-        'detach_replica': ignored if not present or False, if True,
-        specifies the instance is a replica that will be detached from
-        its master
-        'configuration_id': Ignored if not present, if None, detaches an
-        an attached configuration group, if not None, attaches the
-        specified configuration group
-        """
-
         if 'detach_replica' in kwargs and kwargs['detach_replica']:
             LOG.debug("Detaching replica from source.")
             context.notification = notification.DBaaSInstanceDetach(
@@ -342,6 +333,14 @@ class InstanceController(wsgi.Controller):
                                                                   request=req))
                 with StartNotification(context, instance_id=instance.id):
                     instance.unassign_configuration()
+        if 'datastore_version' in kwargs:
+            datastore_version = datastore_models.DatastoreVersion.load(
+                instance.datastore, kwargs['datastore_version'])
+            context.notification = (
+                notification.DBaaSInstanceUpgrade(context, request=req))
+            with StartNotification(context, instance_id=instance.id,
+                                   datastore_version_id=datastore_version.id):
+                instance.upgrade(datastore_version)
         if kwargs:
             instance.update_db(**kwargs)
 
@@ -381,6 +380,10 @@ class InstanceController(wsgi.Controller):
             args['name'] = body['instance']['name']
         if 'configuration' in body['instance']:
             args['configuration_id'] = self._configuration_parse(context, body)
+        if 'datastore_version' in body['instance']:
+            args['datastore_version'] = body['instance'].get(
+                'datastore_version')
+
         self._modify_instance(context, req, instance, **args)
         return wsgi.Result(None, 202)
 

trove/instance/tasks.py

@@ -114,6 +114,7 @@ class InstanceTasks(object):
     SHRINKING_ERROR = InstanceTask(0x58, 'SHRINKING',
                                          'Shrinking Cluster Error.',
                                          is_error=True)
+    UPGRADING = InstanceTask(0x59, 'UPGRADING', 'Upgrading the instance.')
 
 # Dissuade further additions at run-time.
 InstanceTask.__init__ = None

trove/taskmanager/api.py

@@ -198,6 +198,14 @@ class API(object):
 
         self._cast("delete_cluster", self.version_cap, cluster_id=cluster_id)
 
+    def upgrade(self, instance_id, datastore_version_id):
+        LOG.debug("Making async call to upgrade guest to datastore "
+                  "version %s " % datastore_version_id)
+
+        cctxt = self.client.prepare(version=self.version_cap)
+        cctxt.cast(self.context, "upgrade", instance_id=instance_id,
+                   datastore_version_id=datastore_version_id)
+
 
 def load(context, manager=None):
     if manager:

trove/taskmanager/manager.py

@@ -30,6 +30,7 @@ from trove.common import remote
 import trove.common.rpc.version as rpc_version
 from trove.common import server_group as srv_grp
 from trove.common.strategies.cluster import strategy
+from trove.datastore.models import DatastoreVersion
 import trove.extensions.mgmt.instances.models as mgmtmodels
 from trove.instance.tasks import InstanceTasks
 from trove.taskmanager import models
@@ -383,6 +384,12 @@ class Manager(periodic_task.PeriodicTasks):
                                   cluster_config, volume_type, modules,
                                   locality)
 
+    def upgrade(self, context, instance_id, datastore_version_id):
+        instance_tasks = models.BuiltInstanceTasks.load(context, instance_id)
+        datastore_version = DatastoreVersion.load_by_uuid(datastore_version_id)
+        with EndNotification(context):
+            instance_tasks.upgrade(datastore_version)
+
     def update_overrides(self, context, instance_id, overrides):
         instance_tasks = models.BuiltInstanceTasks.load(context, instance_id)
         instance_tasks.update_overrides(overrides)

trove/taskmanager/models.py

@@ -336,32 +336,6 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
 
         LOG.debug("End _delete_resource for instance %s" % self.id)
 
-    def _get_injected_files(self, datastore_manager):
-        injected_config_location = CONF.get('injected_config_location')
-        guest_info = CONF.get('guest_info')
-
-        if ('/' in guest_info):
-            # Set guest_info_file to exactly guest_info from the conf file.
-            # This should be /etc/guest_info for pre-Kilo compatibility.
-            guest_info_file = guest_info
-        else:
-            guest_info_file = os.path.join(injected_config_location,
-                                           guest_info)
-
-        files = {guest_info_file: (
-            "[DEFAULT]\n"
-            "guest_id=%s\n"
-            "datastore_manager=%s\n"
-            "tenant_id=%s\n"
-            % (self.id, datastore_manager, self.tenant_id))}
-
-        if os.path.isfile(CONF.get('guest_config')):
-            with open(CONF.get('guest_config'), "r") as f:
-                files[os.path.join(injected_config_location,
-                                   "trove-guestagent.conf")] = f.read()
-
-        return files
-
     def wait_for_instance(self, timeout, flavor):
         # Make sure the service becomes active before sending a usage
         # record to avoid over billing a customer for an instance that
@@ -421,7 +395,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 LOG.debug("Successfully created security group for "
                           "instance: %s" % self.id)
 
-        files = self._get_injected_files(datastore_manager)
+        files = self.get_injected_files(datastore_manager)
         cinder_volume_type = volume_type or CONF.cinder_volume_type
         if use_heat:
             volume_info = self._create_server_volume_heat(
@@ -1420,6 +1394,58 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
         datastore_status.status = rd_instance.ServiceStatuses.PAUSED
         datastore_status.save()
 
+    def upgrade(self, datastore_version):
+        LOG.debug("Upgrading instance %s to new datastore version %s",
+                  self, datastore_version)
+
+        def server_finished_rebuilding():
+            self.refresh_compute_server_info()
+            return not self.server_status_matches(['REBUILD'])
+
+        try:
+            upgrade_info = self.guest.pre_upgrade()
+
+            if self.volume_id:
+                volume = self.volume_client.volumes.get(self.volume_id)
+                volume_device = self._fix_device_path(
+                    volume.attachments[0]['device'])
+
+            injected_files = self.get_injected_files(
+                datastore_version.manager)
+            LOG.debug("Rebuilding instance %(instance)s with image %(image)s.",
+                      {'instance': self, 'image': datastore_version.image_id})
+            self.server.rebuild(datastore_version.image_id,
+                                files=injected_files)
+            utils.poll_until(
+                server_finished_rebuilding,
+                sleep_time=2, time_out=600)
+            if not self.server_status_matches(['ACTIVE']):
+                raise TroveError(_("Instance %(instance)s failed to "
+                                   "upgrade to %(datastore_version)s")
+                                 % {'instance': self,
+                                    'datastore_version': datastore_version})
+
+            if volume:
+                upgrade_info['device'] = volume_device
+
+            self.guest.post_upgrade(upgrade_info)
+
+            self.reset_task_status()
+
+        except Exception as e:
+            LOG.exception(e)
+            err = inst_models.InstanceTasks.BUILDING_ERROR_SERVER
+            self.update_db(task_status=err)
+            raise e
+
+    # Some cinder drivers appear to return "vdb" instead of "/dev/vdb".
+    # We need to account for that.
+    def _fix_device_path(self, device):
+        if device.startswith("/dev"):
+            return device
+        else:
+            return "/dev/%s" % device
+
 
 class BackupTasks(object):
     @classmethod

trove/tests/fakes/nova.py

@@ -17,6 +17,7 @@ from novaclient import exceptions as nova_exceptions
 from oslo_log import log as logging
 
 from trove.common.exception import PollTimeOut
+from trove.common.i18n import _
 from trove.common import instance as rd_instance
 from trove.tests.fakes.common import authorize
 

trove/tests/int_tests.py

@@ -42,6 +42,7 @@ from trove.tests.scenario.groups import instance_actions_group
 from trove.tests.scenario.groups import instance_create_group
 from trove.tests.scenario.groups import instance_delete_group
 from trove.tests.scenario.groups import instance_error_create_group
+from trove.tests.scenario.groups import instance_upgrade_group
 from trove.tests.scenario.groups import module_group
 from trove.tests.scenario.groups import negative_cluster_actions_group
 from trove.tests.scenario.groups import replication_group
@@ -146,6 +147,9 @@ instance_create_groups.extend([instance_create_group.GROUP,
 instance_error_create_groups = list(base_groups)
 instance_error_create_groups.extend([instance_error_create_group.GROUP])
 
+instance_upgrade_groups = list(instance_create_groups)
+instance_upgrade_groups.extend([instance_upgrade_group.GROUP])
+
 backup_groups = list(instance_create_groups)
 backup_groups.extend([groups.BACKUP,
                       groups.BACKUP_INST])
@@ -204,6 +208,7 @@ register(["guest_log"], guest_log_groups)
 register(["instance", "instance_actions"], instance_actions_groups)
 register(["instance_create"], instance_create_groups)
 register(["instance_error_create"], instance_error_create_groups)
+register(["instance_upgrade"], instance_upgrade_groups)
 register(["module"], module_groups)
 register(["module_create"], module_create_groups)
 register(["replication"], replication_groups)
@@ -228,8 +233,8 @@ register(["postgresql_supported"], common_groups,
          backup_incremental_groups, replication_groups)
 register(["mysql_supported", "percona_supported"], common_groups,
          backup_groups, configuration_groups, database_actions_groups,
-         replication_promote_groups, root_actions_groups, user_actions_groups,
-         backup_incremental_groups)
+         replication_promote_groups, instance_upgrade_groups,
+         root_actions_groups, user_actions_groups, backup_incremental_groups)
 register(["mariadb_supported"], common_groups,
          backup_groups, cluster_actions_groups, configuration_groups,
          database_actions_groups, replication_promote_groups,

trove/tests/scenario/groups/__init__.py

@@ -64,6 +64,10 @@ INST_ACTIONS_RESIZE = "scenario.inst_actions_resize_grp"
 INST_ACTIONS_RESIZE_WAIT = "scenario.inst_actions_resize_wait_grp"
 
 
+# Instance Upgrade Group
+INST_UPGRADE = "scenario.inst_upgrade_grp"
+
+
 # Instance Create Group
 INST_CREATE = "scenario.inst_create_grp"
 INST_CREATE_WAIT = "scenario.inst_create_wait_grp"

trove/tests/scenario/groups/configuration_group.py

@@ -235,9 +235,11 @@ class ConfigurationInstCreateGroup(TestGroup):
       groups=[GROUP, groups.CFGGRP_INST,
               groups.CFGGRP_INST_CREATE_WAIT],
       runs_after_groups=[groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.MODULE_INST_CREATE_WAIT])
 class ConfigurationInstCreateWaitGroup(TestGroup):
     """Test that Instance Configuration Group Create Completes."""
+
     def __init__(self):
         super(ConfigurationInstCreateWaitGroup, self).__init__(
             ConfigurationRunnerFactory.instance())

trove/tests/scenario/groups/instance_actions_group.py

@@ -54,6 +54,7 @@ class InstanceActionsGroup(TestGroup):
 @test(depends_on_groups=[groups.INST_CREATE_WAIT],
       groups=[GROUP, groups.INST_ACTIONS_RESIZE],
       runs_after_groups=[groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.MODULE_INST_CREATE_WAIT,
                          groups.CFGGRP_INST_CREATE_WAIT,
                          groups.BACKUP_CREATE,

trove/tests/scenario/groups/instance_delete_group.py

@@ -33,6 +33,7 @@ class InstanceDeleteRunnerFactory(test_runners.RunnerFactory):
       groups=[GROUP, groups.INST_DELETE],
       runs_after_groups=[groups.INST_INIT_DELETE,
                          groups.INST_ACTIONS,
+                         groups.INST_UPGRADE,
                          groups.INST_ACTIONS_RESIZE_WAIT,
                          groups.BACKUP_INST_DELETE,
                          groups.BACKUP_INC_INST_DELETE,

trove/tests/scenario/groups/instance_upgrade_group.py

@@ -0,0 +1,92 @@
+# Copyright 2015 Tesora Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from proboscis import test
+
+from trove.tests.scenario import groups
+from trove.tests.scenario.groups.test_group import TestGroup
+from trove.tests.scenario.runners import test_runners
+
+
+GROUP = "scenario.instance_upgrade_group"
+
+
+class InstanceUpgradeRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'instance_upgrade_runners'
+    _runner_cls = 'InstanceUpgradeRunner'
+
+
+class UserActionsRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'user_actions_runners'
+    _runner_cls = 'UserActionsRunner'
+
+
+class DatabaseActionsRunnerFactory(test_runners.RunnerFactory):
+
+    _runner_ns = 'database_actions_runners'
+    _runner_cls = 'DatabaseActionsRunner'
+
+
+@test(depends_on_groups=[groups.INST_CREATE_WAIT],
+      groups=[GROUP, groups.INST_UPGRADE],
+      runs_after_groups=[groups.INST_ACTIONS])
+class InstanceUpgradeGroup(TestGroup):
+
+    def __init__(self):
+        super(InstanceUpgradeGroup, self).__init__(
+            InstanceUpgradeRunnerFactory.instance())
+        self.database_actions_runner = DatabaseActionsRunnerFactory.instance()
+        self.user_actions_runner = UserActionsRunnerFactory.instance()
+
+    @test
+    def create_user_databases(self):
+        """Create user databases on an existing instance."""
+        # These databases may be referenced by the users (below) so we need to
+        # create them first.
+        self.database_actions_runner.run_databases_create()
+
+    @test(runs_after=[create_user_databases])
+    def create_users(self):
+        """Create users on an existing instance."""
+        self.user_actions_runner.run_users_create()
+
+    @test(runs_after=[create_users])
+    def instance_upgrade(self):
+        """Upgrade an existing instance."""
+        self.test_runner.run_instance_upgrade()
+
+    @test(depends_on=[instance_upgrade])
+    def show_user(self):
+        """Show created users."""
+        self.user_actions_runner.run_user_show()
+
+    @test(depends_on=[create_users],
+          runs_after=[show_user])
+    def list_users(self):
+        """List the created users."""
+        self.user_actions_runner.run_users_list()
+
+    @test(depends_on=[create_users],
+          runs_after=[list_users])
+    def delete_user(self):
+        """Delete the created users."""
+        self.user_actions_runner.run_user_delete()
+
+    @test(depends_on=[create_user_databases], runs_after=[delete_user])
+    def delete_user_databases(self):
+        """Delete the user databases."""
+        self.database_actions_runner.run_database_delete()

trove/tests/scenario/groups/module_group.py

@@ -374,7 +374,7 @@ class ModuleInstCreateGroup(TestGroup):
 
 @test(depends_on_groups=[groups.MODULE_INST_CREATE],
       groups=[GROUP, groups.MODULE_INST, groups.MODULE_INST_CREATE_WAIT],
-      runs_after_groups=[groups.INST_ACTIONS])
+      runs_after_groups=[groups.INST_ACTIONS, groups.INST_UPGRADE])
 class ModuleInstCreateWaitGroup(TestGroup):
     """Test that Module Instance Create Completes."""
 

trove/tests/scenario/runners/instance_upgrade_runners.py

@@ -0,0 +1,33 @@
+# Copyright 2015 Tesora Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from trove.tests.scenario.runners.test_runners import TestRunner
+
+
+class InstanceUpgradeRunner(TestRunner):
+
+    def __init__(self):
+        super(InstanceUpgradeRunner, self).__init__()
+
+    def run_instance_upgrade(
+            self, expected_states=['UPGRADE', 'ACTIVE'],
+            expected_http_code=202):
+        instance_id = self.instance_info.id
+        self.report.log("Testing upgrade on instance: %s" % instance_id)
+
+        target_version = self.instance_info.dbaas_datastore_version
+        self.auth_client.instances.upgrade(instance_id, target_version)
+        self.assert_instance_action(instance_id, expected_states,
+                                    expected_http_code)

trove/tests/unittests/guestagent/test_dbaas.py

@@ -1548,10 +1548,12 @@ class MySqlAppMockTest(trove_testtools.TestCase):
         utils.execute_with_timeout = self.orig_utils_execute_with_timeout
         super(MySqlAppMockTest, self).tearDown()
 
+    @patch('trove.guestagent.common.configuration.ConfigurationManager'
+           '.refresh_cache')
     @patch.object(mysql_common_service, 'clear_expired_password')
     @patch.object(utils, 'generate_random_password',
                   return_value='some_password')
-    def test_secure_keep_root(self, auth_pwd_mock, clear_pwd_mock):
+    def test_secure_keep_root(self, auth_pwd_mock, clear_pwd_mock, _):
         with patch.object(self.mock_client,
                           'execute', return_value=None) as mock_execute:
             utils.execute_with_timeout = MagicMock(return_value=None)
@@ -1569,10 +1571,12 @@ class MySqlAppMockTest(trove_testtools.TestCase):
                 app._reset_configuration.assert_has_calls(reset_config_calls)
                 self.assertTrue(mock_execute.called)
 
+    @patch('trove.guestagent.common.configuration.ConfigurationManager'
+           '.refresh_cache')
     @patch.object(mysql_common_service, 'clear_expired_password')
     @patch.object(mysql_common_service.BaseMySqlApp,
                   'get_auth_password', return_value='some_password')
-    def test_secure_with_mycnf_error(self, auth_pwd_mock, clear_pwd_mock):
+    def test_secure_with_mycnf_error(self, *args):
         with patch.object(self.mock_client,
                           'execute', return_value=None) as mock_execute:
             with patch.object(operating_system, 'service_discovery',

trove/tests/unittests/instance/test_instance_models.py

@@ -250,6 +250,78 @@ class CreateInstanceTest(trove_testtools.TestCase):
         self.assertIsNotNone(instance)
 
 
+class TestInstanceUpgrade(trove_testtools.TestCase):
+
+    def setUp(self):
+        self.context = trove_testtools.TroveTestContext(self, is_admin=True)
+        util.init_db()
+
+        self.datastore = datastore_models.DBDatastore.create(
+            id=str(uuid.uuid4()),
+            name='test' + str(uuid.uuid4()),
+            default_version_id=str(uuid.uuid4()))
+
+        self.datastore_version1 = datastore_models.DBDatastoreVersion.create(
+            id=self.datastore.default_version_id,
+            name='name' + str(uuid.uuid4()),
+            image_id='old_image',
+            packages=str(uuid.uuid4()),
+            datastore_id=self.datastore.id,
+            manager='test',
+            active=1)
+
+        self.datastore_version2 = datastore_models.DBDatastoreVersion.create(
+            id=str(uuid.uuid4()),
+            name='name' + str(uuid.uuid4()),
+            image_id='new_image',
+            packages=str(uuid.uuid4()),
+            datastore_id=self.datastore.id,
+            manager='test',
+            active=1)
+
+        self.safe_nova_client = models.create_nova_client
+        models.create_nova_client = nova.fake_create_nova_client
+        super(TestInstanceUpgrade, self).setUp()
+
+    def tearDown(self):
+        self.datastore.delete()
+        self.datastore_version1.delete()
+        self.datastore_version2.delete()
+        models.create_nova_client = self.safe_nova_client
+        super(TestInstanceUpgrade, self).tearDown()
+
+    @patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
+    @patch.object(task_api.API, 'upgrade')
+    def test_upgrade(self, task_upgrade):
+        instance_model = DBInstance(
+            InstanceTasks.NONE,
+            id=str(uuid.uuid4()),
+            name="TestUpgradeInstance",
+            datastore_version_id=self.datastore_version1.id)
+        instance_model.set_task_status(InstanceTasks.NONE)
+        instance_model.save()
+        instance_status = InstanceServiceStatus(
+            ServiceStatuses.RUNNING,
+            id=str(uuid.uuid4()),
+            instance_id=instance_model.id)
+        instance_status.save()
+        self.assertIsNotNone(instance_model)
+        instance = models.load_instance(models.Instance, self.context,
+                                        instance_model.id)
+
+        try:
+            instance.upgrade(self.datastore_version2)
+
+            self.assertEqual(self.datastore_version2.id,
+                             instance.db_info.datastore_version_id)
+            self.assertEqual(InstanceTasks.UPGRADING,
+                             instance.db_info.task_status)
+            self.assertTrue(task_upgrade.called)
+        finally:
+            instance_status.delete()
+            instance_model.delete()
+
+
 class TestReplication(trove_testtools.TestCase):
 
     def setUp(self):

trove/tests/unittests/taskmanager/test_api.py

@@ -122,6 +122,14 @@ class ApiTest(trove_testtools.TestCase):
                                 ('Could not transform %s' % flavor),
                                 self.api._transform_obj, flavor)
 
+    def test_upgrade(self):
+        self.api.upgrade('some-instance-id', 'some-datastore-version')
+
+        self._verify_rpc_prepare_before_cast()
+        self._verify_cast('upgrade',
+                          instance_id='some-instance-id',
+                          datastore_version_id='some-datastore-version')
+
 
 class TestAPI(trove_testtools.TestCase):
 

trove/tests/unittests/taskmanager/test_models.py

@@ -18,6 +18,7 @@ import uuid
 
 from cinderclient import exceptions as cinder_exceptions
 import cinderclient.v2.client as cinderclient
+from cinderclient.v2 import volumes as cinderclient_volumes
 from mock import Mock, MagicMock, patch, PropertyMock, call
 from novaclient import exceptions as nova_exceptions
 import novaclient.v2.flavors
@@ -217,6 +218,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
         self.task_models_conf_patch = patch('trove.taskmanager.models.CONF')
         self.task_models_conf_mock = self.task_models_conf_patch.start()
         self.addCleanup(self.task_models_conf_patch.stop)
+        self.inst_models_conf_patch = patch('trove.instance.models.CONF')
+        self.inst_models_conf_mock = self.inst_models_conf_patch.start()
+        self.addCleanup(self.inst_models_conf_patch.stop)
 
     def tearDown(self):
         super(FreshInstanceTasksTest, self).tearDown()
@@ -252,9 +256,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             else:
                 return ''
 
-        self.task_models_conf_mock.get.side_effect = fake_conf_getter
+        self.inst_models_conf_mock.get.side_effect = fake_conf_getter
         # execute
-        files = self.freshinstancetasks._get_injected_files("test")
+        files = self.freshinstancetasks.get_injected_files("test")
         # verify
         self.assertTrue(
             '/etc/trove/conf.d/guest_info.conf' in files)
@@ -275,9 +279,9 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
             else:
                 return ''
 
-        self.task_models_conf_mock.get.side_effect = fake_conf_getter
+        self.inst_models_conf_mock.get.side_effect = fake_conf_getter
         # execute
-        files = self.freshinstancetasks._get_injected_files("test")
+        files = self.freshinstancetasks.get_injected_files("test")
         # verify
         self.assertTrue(
             '/etc/guest_info' in files)
@@ -396,7 +400,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
     @patch.object(BaseInstance, 'update_db')
     @patch.object(backup_models.Backup, 'get_by_id')
     @patch.object(taskmanager_models.FreshInstanceTasks, 'report_root_enabled')
-    @patch.object(taskmanager_models.FreshInstanceTasks, '_get_injected_files')
+    @patch.object(taskmanager_models.FreshInstanceTasks, 'get_injected_files')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_secgroup')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_build_volume_info')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_server')
@@ -417,7 +421,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase):
 
     @patch.object(BaseInstance, 'update_db')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_dns_entry')
-    @patch.object(taskmanager_models.FreshInstanceTasks, '_get_injected_files')
+    @patch.object(taskmanager_models.FreshInstanceTasks, 'get_injected_files')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_server')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_create_secgroup')
     @patch.object(taskmanager_models.FreshInstanceTasks, '_build_volume_info')
@@ -691,6 +695,10 @@ class BuiltInstanceTasksTest(trove_testtools.TestCase):
                                                    True)
         stub_flavor_manager.get = MagicMock(return_value=nova_flavor)
 
+        self.instance_task._volume_client = MagicMock(spec=cinderclient)
+        self.instance_task._volume_client.volumes = Mock(
+            spec=cinderclient_volumes.VolumeManager)
+
         answers = (status for status in
                    self.get_inst_service_status('inst_stat-id',
                                                 [ServiceStatuses.SHUTDOWN,
@@ -917,6 +925,36 @@ class BuiltInstanceTasksTest(trove_testtools.TestCase):
         self.instance_task.demote_replication_master()
         self.instance_task._guest.demote_replication_master.assert_any_call()
 
+    @patch.multiple(taskmanager_models.BuiltInstanceTasks,
+                    get_injected_files=Mock(return_value="the-files"))
+    def test_upgrade(self, *args):
+        pre_rebuild_server = self.instance_task.server
+        dsv = Mock(image_id='foo_image')
+        mock_volume = Mock(attachments=[{'device': '/dev/mock_dev'}])
+        with patch.object(self.instance_task._volume_client.volumes, "get",
+                          Mock(return_value=mock_volume)):
+            mock_server = Mock(status='ACTIVE')
+            with patch.object(self.instance_task._nova_client.servers,
+                              'get', Mock(return_value=mock_server)):
+                with patch.multiple(self.instance_task._guest,
+                                    pre_upgrade=Mock(return_value={}),
+                                    post_upgrade=Mock()):
+                    self.instance_task.upgrade(dsv)
+
+                    self.instance_task._guest.pre_upgrade.assert_called_with()
+                    pre_rebuild_server.rebuild.assert_called_with(
+                        dsv.image_id, files="the-files")
+                    self.instance_task._guest.post_upgrade.assert_called_with(
+                        mock_volume.attachments[0])
+
+    def test_fix_device_path(self):
+        self.assertEqual("/dev/vdb", self.instance_task.
+                         _fix_device_path("vdb"))
+        self.assertEqual("/dev/dev", self.instance_task.
+                         _fix_device_path("dev"))
+        self.assertEqual("/dev/vdb/dev", self.instance_task.
+                         _fix_device_path("vdb/dev"))
+
 
 class BackupTasksTest(trove_testtools.TestCase):