From 4c83bb8862c987cb0f88ee35c5f4b2cd20d86844 Mon Sep 17 00:00:00 2001 From: Bo Tran Date: Wed, 24 May 2023 17:23:55 +0700 Subject: [PATCH] Separate backup docker image for each database version Co-Authored-By: wu.chunyang Story: #2010770 Task: #48087 Change-Id: I08063748e15de6767b437aa443311d41e25ed578 --- doc/source/admin/run_trove_in_production.rst | 45 +++++++++- .../separate-backup-docker-image-884165.yaml | 6 ++ trove/common/cfg.py | 24 +++-- trove/guestagent/datastore/mysql/service.py | 17 ++-- .../guestagent/datastore/postgres/service.py | 4 +- trove/guestagent/datastore/service.py | 26 +++++- .../guestagent/datastore/test_service.py | 88 +++++++++++++++++++ 7 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 releasenotes/notes/separate-backup-docker-image-884165.yaml create mode 100644 trove/tests/unittests/guestagent/datastore/test_service.py diff --git a/doc/source/admin/run_trove_in_production.rst b/doc/source/admin/run_trove_in_production.rst index 60dc1ff942..9207e4ab24 100644 --- a/doc/source/admin/run_trove_in_production.rst +++ b/doc/source/admin/run_trove_in_production.rst @@ -301,7 +301,50 @@ Some config options specifically for trove guest agent: [mysql] docker_image = your-registry/your-repo/mysql - backup_docker_image = your-registry/your-repo/db-backup-mysql:1.1.0 + backup_docker_image = your-registry/your-repo/db-backup-mysql + +Make Trove work with multiple versions for each datastore +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When Trove do a backup/restore actions, The Trove guest agent pulls container +images with tags matching the datastore version of the database instance running. +To Ensure the trove guest agent can run backup/restore properly, you need to ensure +the images with the proper tags already exists in the registry. +Such as: +If your datastore manager is 'mariadb' and its name is 'MariaDB', +and it has 2 datastore versions: + + .. code-block:: bash + + openstack datastore version list MariaDB + +--------------------------------------+------+---------+ + | ID | Name | Version | + +--------------------------------------+------+---------+ + | 550aebf7-df97-49f1-bf24-7cd7b69fa365 | 10.3 | 10.3 | + | ee988cc3-bb30-4aaf-9837-e90a34f60d37 | 10.4 | 10.4 | + +--------------------------------------+------+---------+ + +Configure the ``backup_docker_image`` options like following: + +.. path /etc/trove/trove-guestagent.conf +.. code-block:: ini + + [mariadb] + # Database docker image. (string value) + docker_image = your-registry/your-repo/db-mariadb + + # The docker image used for backup and restore. (string value) + backup_docker_image = your-registry/your-repo/db-backup-mariadb + +.. note:: + + Do not configure the image tag for the image. because if the image doesn't + contain the tag, Trove will use the datastore version as the tag. + +Administrators need to ensure that the Docker backup image has 2 tags (10.3 & 10.4) +in docker registry. For example: +your-registry/your-repo/db-backup-mariadb:10.3 & your-registry/your-repo/db-backup-mariadb:10.4 + +Finally, when trove-guestagent does backup/restore, it will pull this image with the tag equals datastore version. Initialize Trove Database ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/releasenotes/notes/separate-backup-docker-image-884165.yaml b/releasenotes/notes/separate-backup-docker-image-884165.yaml new file mode 100644 index 0000000000..9c33658079 --- /dev/null +++ b/releasenotes/notes/separate-backup-docker-image-884165.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for multiple datastore versions for each datastore. + more details: + `Story 2010770 `__ \ No newline at end of file diff --git a/trove/common/cfg.py b/trove/common/cfg.py index b837dab295..4a27e2e38c 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -660,10 +660,12 @@ mysql_opts = [ help='Database docker image.' ), cfg.StrOpt( - 'backup_docker_image', default='openstacktrove/db-backup-mysql:1.1.0', - help='The docker image used for backup and restore. For mysql, ' - 'the minor version is added to the image name as a suffix before ' - 'creating container, e.g. openstacktrove/db-backup-mysql5.7:1.0.0' + 'backup_docker_image', + sample_default='your-registry/your-repo/db-backup-mysql', + help='The docker image used for backup and restore. Trove will uses' + 'datastore version as the image tag, for example: ' + 'your-registry/your-repo/db-backup-mysql:5.7 is used for mysql' + 'datastore with version 5.7' ), ] @@ -1112,8 +1114,11 @@ postgresql_opts = [ ), cfg.StrOpt( 'backup_docker_image', - default='openstacktrove/db-backup-postgresql:1.1.2', - help='The docker image used for backup and restore.' + sample_default='your-registry/your-repo/db-backup-postgresql', + help='The docker image used for backup and restore. Trove will uses' + 'datastore version as the image tag, for example: ' + 'your-registry/your-repo/db-backup-postgresql:12 is used for' + 'postgresql datastore with version 12' ), cfg.BoolOpt('icmp', default=False, help='Whether to permit ICMP.', @@ -1437,8 +1442,11 @@ mariadb_opts = [ ), cfg.StrOpt( 'backup_docker_image', - default='openstacktrove/db-backup-mariadb:1.1.0', - help='The docker image used for backup and restore.' + sample_default='your-registry/your-repo/db-backup-mariadb', + help='The docker image used for backup and restore. Trove will uses' + 'datastore version as the image tag, for example: ' + 'your-registry/your-repo/db-backup-mariadb:10.3 is used for ' + 'postgresql datastore with version 10.3' ), ] diff --git a/trove/guestagent/datastore/mysql/service.py b/trove/guestagent/datastore/mysql/service.py index 78a371cd72..7401706ec3 100644 --- a/trove/guestagent/datastore/mysql/service.py +++ b/trove/guestagent/datastore/mysql/service.py @@ -65,15 +65,18 @@ class MySqlApp(service.BaseMySqlApp): For example, this method converts openstacktrove/db-backup-mysql:1.0.0 to openstacktrove/db-backup-mysql5.7:1.0.0 + + **deprecated**: this function is for backward compatibility. """ image = cfg.get_configuration_property('backup_docker_image') - name, tag = image.rsplit(':', 1) - - # Get minor version - cur_ver = semantic_version.Version.coerce(CONF.datastore_version) - minor_ver = f"{cur_ver.major}.{cur_ver.minor}" - - return f"{name}{minor_ver}:{tag}" + if not self._image_has_tag(image): + return super().get_backup_image() + else: + name, tag = image.rsplit(':', 1) + # Get minor version + cur_ver = semantic_version.Version.coerce(CONF.datastore_version) + minor_ver = f"{cur_ver.major}.{cur_ver.minor}" + return f"{name}{minor_ver}:{tag}" def get_backup_strategy(self): """Get backup strategy. diff --git a/trove/guestagent/datastore/postgres/service.py b/trove/guestagent/datastore/postgres/service.py index 721ecfcd5a..7d5fbfb4f2 100644 --- a/trove/guestagent/datastore/postgres/service.py +++ b/trove/guestagent/datastore/postgres/service.py @@ -263,8 +263,8 @@ class PgSqlApp(service.BaseDbApp): def restore_backup(self, context, backup_info, restore_location): backup_id = backup_info['id'] storage_driver = CONF.storage_strategy - backup_driver = cfg.get_configuration_property('backup_strategy') - image = cfg.get_configuration_property('backup_docker_image') + backup_driver = self.get_backup_strategy() + image = self.get_backup_image() name = 'db_restore' volumes = { '/var/lib/postgresql/data': { diff --git a/trove/guestagent/datastore/service.py b/trove/guestagent/datastore/service.py index afa4d234de..2134da2f9e 100644 --- a/trove/guestagent/datastore/service.py +++ b/trove/guestagent/datastore/service.py @@ -415,8 +415,32 @@ class BaseDbApp(object): self.reset_configuration(config_contents) self.start_db(update_db=True, ds_version=ds_version) + @staticmethod + def _image_has_tag(image): + """ + Whether docker_image being config with tag + "example.domain:5000/repo/image_name:tag", + "example.domain:5000/repo/image-name:tag", + "example.domain:5000/repo/image-name:tag_tag", + "example.domain:5000/repo/image_name:tag-tag", + "example.domain:5000/repo/image-name", + "example.domain:5000/repo/image_name", + "example.domain:5000:5000/repo/image-name", + "example.domain/repo/image-name", + "example.domain/repo/image-name:tag" + + Returns: + - True if match + """ + return image.split('/')[-1].find(':') > 0 + def get_backup_image(self): - return cfg.get_configuration_property('backup_docker_image') + image = cfg.get_configuration_property('backup_docker_image') + if not self._image_has_tag(image): + ds_version = CONF.datastore_version + image = (f'{image}:latest' if not ds_version else + f'{image}:{ds_version}') + return image def get_backup_strategy(self): return cfg.get_configuration_property('backup_strategy') diff --git a/trove/tests/unittests/guestagent/datastore/test_service.py b/trove/tests/unittests/guestagent/datastore/test_service.py new file mode 100644 index 0000000000..ed8922fef6 --- /dev/null +++ b/trove/tests/unittests/guestagent/datastore/test_service.py @@ -0,0 +1,88 @@ +# Copyright 2023 BizflyCloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest import mock + +from trove.common import cfg +from trove.guestagent.datastore.mariadb import service +from trove.guestagent.datastore.mysql import service as mysql_service +from trove.guestagent.datastore import service as base_service +from trove.tests.unittests import trove_testtools + + +CONF = cfg.CONF + + +class TestService(trove_testtools.TestCase): + def setUp(self): + super(TestService, self).setUp() + _docker_client = mock.MagicMock() + status = mock.MagicMock() + self.app = service.MariaDBApp(_docker_client, status) + self.mysql_app = mysql_service.MySqlApp(_docker_client, status) + + def test_get_backup_image_with_tag(self): + self.patch_datastore_manager('mariadb') + CONF.set_override('backup_docker_image', + 'example.domain/repo/mariadb:tag', 'mariadb') + image = self.app.get_backup_image() + self.assertEqual(CONF.mariadb.backup_docker_image, image) + + def test_get_backup_image_without_tag(self): + self.patch_datastore_manager('mariadb') + CONF.set_override('backup_docker_image', + 'example.domain/repo/mariadb', 'mariadb') + self.patch_conf_property('datastore_version', '10.4') + image = self.app.get_backup_image() + _img = f'{CONF.mariadb.backup_docker_image}:{CONF.datastore_version}' + self.assertEqual(_img, image) + + def test_mysql_backup_image_with_tag(self): + self.patch_datastore_manager('mysql') + CONF.set_override('backup_docker_image', + 'example.domain/repo/mysql:1.1.0', 'mysql') + self.patch_conf_property('datastore_version', '5.7') + image = self.mysql_app.get_backup_image() + self.assertEqual(image, "example.domain/repo/mysql5.7:1.1.0") + + def test_mysql_backup_image_without_tag(self): + self.patch_datastore_manager('mysql') + CONF.set_override('backup_docker_image', + 'example.domain/repo/mysql', 'mysql') + self.patch_conf_property('datastore_version', '5.7') + image = self.mysql_app.get_backup_image() + self.assertEqual(image, "example.domain/repo/mysql:5.7") + + def test_image_has_tag(self): + fake_values = [ + "example.domain:5000/repo/image_name:tag", + "example.domain:5000/repo/image-name:tag_tag", + "example.domain:5000/repo/image_name:tag-tag", + "example.domain:5000/repo/image-name", + "example.domain:5000/repo/image_name", + "example.domain/repo/image-name", + "example.domain/repo/image-name:tag"] + self.assertTrue( + base_service.BaseDbApp._image_has_tag(fake_values[0])) + self.assertTrue( + base_service.BaseDbApp._image_has_tag(fake_values[1])) + self.assertTrue( + base_service.BaseDbApp._image_has_tag(fake_values[2])) + self.assertFalse( + base_service.BaseDbApp._image_has_tag(fake_values[3])) + self.assertFalse( + base_service.BaseDbApp._image_has_tag(fake_values[4])) + self.assertFalse( + base_service.BaseDbApp._image_has_tag(fake_values[5])) + self.assertTrue( + base_service.BaseDbApp._image_has_tag(fake_values[6]))