From 2fd324357b0b953b049a1acbc9e59ffe45ab883a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Godek?= Date: Wed, 3 Apr 2019 13:09:51 +0200 Subject: [PATCH] Add Redis datastore upgrade This patch adds support for upgrading Redis datastore - both single instance and cluster. It is achievied in a similar way to MySQL/Mariadb. Steps are: + For each node do: + stop database + preserve configuration files, + unmount data volume, + create new instance with new datastore, + enter restarting mode + mount data volume, + restore configuration files, + reload datastore Note that due to the Redis Cluster design and Trove approach (all nodes are master nodes) we do not perform real rolling upgrade - some Redis slots may be unavailable while upgrading node associated with them. This patch adds also a user documentation about upgrading datastores using Redis as an example. Story: #2005421 Task: #30443 Co-Authored-By: Kasper Hasior Change-Id: I60cddb3a41d2cc7024cbec3d2fd2038d79446507 Signed-off-by: Kasper Hasior --- doc/source/user/index.rst | 2 + doc/source/user/upgrade-cluster-datastore.rst | 83 ++++++++++++++++ doc/source/user/upgrade-datastore.rst | 96 +++++++++++++++++++ .../cluster/experimental/redis/api.py | 3 + .../cluster/experimental/redis/taskmanager.py | 3 + .../datastore/experimental/redis/manager.py | 45 +++++++++ 6 files changed, 232 insertions(+) create mode 100644 doc/source/user/upgrade-cluster-datastore.rst create mode 100644 doc/source/user/upgrade-datastore.rst diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index e65cd5d65c..8c104ceca5 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -17,3 +17,5 @@ handling complex administrative tasks. manage-db-config.rst set-up-replication.rst set-up-clustering.rst + upgrade-datastore.rst + upgrade-cluster-datastore.rst diff --git a/doc/source/user/upgrade-cluster-datastore.rst b/doc/source/user/upgrade-cluster-datastore.rst new file mode 100644 index 0000000000..a11ea7631e --- /dev/null +++ b/doc/source/user/upgrade-cluster-datastore.rst @@ -0,0 +1,83 @@ +========================= +Upgrade cluster datastore +========================= + +Upgrading datastore for cluster instances is very similar to upgrading +a single instance. + +Trove tries to perform a rolling upgrade so that there won't be any +downtime. However, it is not always possible and, for example, in case +of Redis upgrade, some of its slots may be temporarily unavailable. + +Trove strategy upgrades every instance in the entire cluster one by +one. Upgrading is finished once all instances are upgraded. + +Please check the guide for datastore upgrade to check prerequisistes. + +This example shows you how to upgrade Redis datastore (version 3.2.6) +for a cluster. + +Upgrading cluster +~~~~~~~~~~~~~~~~~ + +#. **Check cluster task** + + Use :command:`openstack database cluster list` to check whether the + task of your cluster is NONE. + + .. code-block:: console + + $openstack database cluster list + +--------------------------------------+---------------+-----------+-------------------+-----------+ + | ID | Name | Datastore | Datastore Version | Task Name | + +--------------------------------------+---------------+-----------+-------------------+-----------+ + | 05f2e7b7-8dac-453f-ad5d-38195cd5718f | redis_cluster | redis | 3.2.6 | NONE | + +--------------------------------------+---------------+-----------+-------------------+-----------+ + +#. **Check if target version is available** + + Use :command:`openstack datastore version list` to list all + available versions your datastore. + + .. code-block:: console + + $openstack datastore version list redis + +--------------------------------------+-------+ + | ID | Name | + +--------------------------------------+-------+ + | 483debec-b7c3-4167-ab1d-1765795ed7eb | 3.2.6 | + | 507f666e-193c-4194-9d9d-da8342dcb4f1 | 3.2.7 | + +--------------------------------------+-------+ + +#. **Run cluster-upgrade** + + Use :command:`openstack database cluster upgrade` command to + upgrade your datastore for the selected instance. + + .. code-block:: console + + $openstack database cluster upgrade 05f2e7b7-8dac-453f-ad5d-38195cd5718f 3.2.7 + +#. **Wait until task changes from UPGRADING_CLUSTER to NONE** + + You can use :command:`openstack database cluster list` to check the + current task. + + .. code-block:: console + + $openstack database cluster list + +--------------------------------------+---------------+-----------+-------------------+-----------+ + | ID | Name | Datastore | Datastore Version | Task Name | + +--------------------------------------+---------------+-----------+-------------------+-----------+ + | 05f2e7b7-8dac-453f-ad5d-38195cd5718f | redis_cluster | redis | 3.2.7 | NONE | + +--------------------------------------+---------------+-----------+-------------------+-----------+ + +Other clusters +~~~~~~~~~~~~~~~ + +Upgrade for other clusters works in the same way. Currently Trove +supports upgrades for the following cluster datastores: + +- MySQL. +- MariaDB. +- Redis. diff --git a/doc/source/user/upgrade-datastore.rst b/doc/source/user/upgrade-datastore.rst new file mode 100644 index 0000000000..dd494e2d2b --- /dev/null +++ b/doc/source/user/upgrade-datastore.rst @@ -0,0 +1,96 @@ +================= +Upgrade datastore +================= + +You can upgrade your datastore version. When you perform an upgrade, +the system automatically manages data and configuration files of your +database. + +To perform datastore upgrade, you need: + +- A supported OS image with the target datastore version. + +- A Trove database instance to be upgrade. + +This example shows you how to upgrade Redis datastore (version 3.2.6) +for a single instance database. + +.. note:: + + **Before** you begin make sure that: + + - Your target datastore is binary compatible with the current + datastore. Each database provider has its own compatibilty + policy. Usually there shouldn't be any problem when + performing an upgrade within minor versions. + + - You **do not** downgrade your datastore. + + - Target versions is supported by Trove. For instance, Trove + doesn't support Cassandra >=2.2 at this moment so you + shouldn't perform an upgrade from 2.1 to 2.2. + +Upgrading datastore +~~~~~~~~~~~~~~~~~~~ + +#. **Check instance status** + + Use :command:`openstack database instance list` to check whether the + status of your instance is ACTIVE. + + .. code-block:: console + + $openstack database instance list + +--------------------------------------+------------+-----------+-------------------+--------+-----------+------+-----------+ + | ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | + +--------------------------------------+------------+-----------+-------------------+--------+-----------+------+-----------+ + | 55411e95-1670-497f-8d92-0179f3b4fdd4 | redis_test | redis | 3.2.6 | ACTIVE | 6 | 5 | RegionOne | + +--------------------------------------+------------+-----------+-------------------+--------+-----------+------+-----------+ + +#. **Check if target version is available** + + Use :command:`openstack datastore version list` command to list + all available versions your datastore. + + .. code-block:: console + + $openstack datastore version list redis + +--------------------------------------+-------+ + | ID | Name | + +--------------------------------------+-------+ + | 483debec-b7c3-4167-ab1d-1765795ed7eb | 3.2.6 | + | 507f666e-193c-4194-9d9d-da8342dcb4f1 | 3.2.7 | + +--------------------------------------+-------+ + +#. **Run upgrade** + + Use :command:`openstack database instance` command to upgrade your + datastore for the selected instance. + + .. code-block:: console + + $openstack database instance 55411e95-1670-497f-8d92-0179f3b4fdd4 3.2.7 + +#. **Wait until status changes from UPGRADE to ACTIVE** + + You can use :command:`openstack database instance list` to check the + current status. + + .. code-block:: console + + $openstack database instance list + +--------------------------------------+------------+-----------+-------------------+---------+-----------+------+-----------+ + | ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | + +--------------------------------------+------------+-----------+-------------------+---------+-----------+------+-----------+ + | 55411e95-1670-497f-8d92-0179f3b4fdd4 | redis_test | redis | 3.2.7 | UPGRADE | 6 | 5 | RegionOne | + +--------------------------------------+------------+-----------+-------------------+---------+-----------+------+-----------+ + +Other datastores +~~~~~~~~~~~~~~~~ + +Upgrade for other datastores works in the same way. Currently Trove +supports upgrades for the following datastores: + +- MySQL, +- MariaDB, +- Redis. diff --git a/trove/common/strategies/cluster/experimental/redis/api.py b/trove/common/strategies/cluster/experimental/redis/api.py index af1a303e35..fccf99b88c 100644 --- a/trove/common/strategies/cluster/experimental/redis/api.py +++ b/trove/common/strategies/cluster/experimental/redis/api.py @@ -125,6 +125,9 @@ class RedisCluster(models.Cluster): return RedisCluster(context, db_info, datastore, datastore_version) + def upgrade(self, datastore_version): + self.rolling_upgrade(datastore_version) + def grow(self, instances): LOG.debug("Growing cluster.") diff --git a/trove/common/strategies/cluster/experimental/redis/taskmanager.py b/trove/common/strategies/cluster/experimental/redis/taskmanager.py index a82d65d8a9..5e1aa56c7a 100644 --- a/trove/common/strategies/cluster/experimental/redis/taskmanager.py +++ b/trove/common/strategies/cluster/experimental/redis/taskmanager.py @@ -154,6 +154,9 @@ class RedisClusterTasks(task_models.ClusterTasks): LOG.debug("End grow_cluster for id: %s.", cluster_id) + def upgrade_cluster(self, context, cluster_id, datastore_version): + self.rolling_upgrade_cluster(context, cluster_id, datastore_version) + class RedisTaskManagerAPI(task_api.API): diff --git a/trove/guestagent/datastore/experimental/redis/manager.py b/trove/guestagent/datastore/experimental/redis/manager.py index 083cbae481..a36e086e43 100644 --- a/trove/guestagent/datastore/experimental/redis/manager.py +++ b/trove/guestagent/datastore/experimental/redis/manager.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + from oslo_log import log as logging from trove.common import exception @@ -109,6 +111,49 @@ class Manager(manager.Manager): if snapshot: self.attach_replica(context, snapshot, snapshot['config']) + def pre_upgrade(self, context): + mount_point = self._app.get_working_dir() + save_etc_dir = "%s/etc" % mount_point + home_save = "%s/trove_user" % mount_point + + self._app.status.begin_restart() + self._app.stop_db() + + operating_system.copy("%s/." % system.REDIS_CONF_DIR, save_etc_dir, + preserve=True, as_root=True) + + operating_system.copy("%s/." % os.path.expanduser('~'), home_save, + preserve=True, as_root=True) + + self.unmount_volume(context, mount_point=mount_point) + + return { + 'mount_point': mount_point, + 'save_etc_dir': save_etc_dir, + 'home_save': home_save + } + + def post_upgrade(self, context, upgrade_info): + self._app.stop_db() + + if 'device' in upgrade_info: + self.mount_volume(context, mount_point=upgrade_info['mount_point'], + device_path=upgrade_info['device'], + write_to_fstab=True) + operating_system.chown(path=upgrade_info['mount_point'], + user=system.REDIS_OWNER, + group=system.REDIS_OWNER, + recursive=True, as_root=True) + + self._restore_home_directory(upgrade_info['home_save']) + + self._restore_directory(upgrade_info['save_etc_dir'], + system.REDIS_CONF_DIR) + + self._app = service.RedisApp() + self._app.start_db() + self._app.status.end_restart() + def restart(self, context): """ Restart this redis instance.