Merge "Separate backup docker image for each database version"

doc/source/admin/run_trove_in_production.rst

@@ -301,7 +301,50 @@ Some config options specifically for trove guest agent:
 
       [mysql]
       docker_image = your-registry/your-repo/mysql
-      backup_docker_image = your-registry/your-repo/db-backup-mysql:1.1.0
+      backup_docker_image = your-registry/your-repo/db-backup-mysql
+
+Make Trove work with multiple versions for each datastore
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+When Trove do a backup/restore actions, The Trove guest agent pulls container
+images with tags matching the datastore version of the database instance running.
+To Ensure the trove guest agent can run backup/restore properly, you need to ensure
+the images with the proper tags already exists in the registry.
+Such as:
+If your datastore manager is 'mariadb' and its name is 'MariaDB',
+and it has 2 datastore versions:
+
+  .. code-block:: bash
+
+      openstack datastore version list MariaDB
+      +--------------------------------------+------+---------+
+      | ID                                   | Name | Version |
+      +--------------------------------------+------+---------+
+      | 550aebf7-df97-49f1-bf24-7cd7b69fa365 | 10.3 | 10.3    |
+      | ee988cc3-bb30-4aaf-9837-e90a34f60d37 | 10.4 | 10.4    |
+      +--------------------------------------+------+---------+
+
+Configure the ``backup_docker_image`` options like following:
+
+.. path /etc/trove/trove-guestagent.conf
+.. code-block:: ini
+
+      [mariadb]
+      # Database docker image. (string value)
+      docker_image = your-registry/your-repo/db-mariadb
+
+      # The docker image used for backup and restore. (string value)
+      backup_docker_image = your-registry/your-repo/db-backup-mariadb
+
+.. note::
+
+    Do not configure the image tag for the image. because if the image doesn't
+    contain the tag, Trove will use the datastore version as the tag.
+
+Administrators need to ensure that the Docker backup image has 2 tags (10.3 & 10.4)
+in docker registry. For example:
+your-registry/your-repo/db-backup-mariadb:10.3 & your-registry/your-repo/db-backup-mariadb:10.4
+
+Finally, when trove-guestagent does backup/restore, it will pull this image with the tag equals datastore version.
 
 Initialize Trove Database
 ~~~~~~~~~~~~~~~~~~~~~~~~~

releasenotes/notes/separate-backup-docker-image-884165.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Add support for multiple datastore versions for each datastore.
+    more details:
+    `Story 2010770 <https://storyboard.openstack.org/#!/story/2010770>`__

trove/common/cfg.py

@@ -660,10 +660,12 @@ mysql_opts = [
         help='Database docker image.'
     ),
     cfg.StrOpt(
-        'backup_docker_image', default='openstacktrove/db-backup-mysql:1.1.0',
+        'backup_docker_image',
-        help='The docker image used for backup and restore. For mysql, '
+        sample_default='your-registry/your-repo/db-backup-mysql',
-             'the minor version is added to the image name as a suffix before '
+        help='The docker image used for backup and restore. Trove will uses'
-             'creating container, e.g. openstacktrove/db-backup-mysql5.7:1.0.0'
+             'datastore version as the image tag, for example: '
+             'your-registry/your-repo/db-backup-mysql:5.7 is used for mysql'
+             'datastore with version 5.7'
     ),
 ]
 
@@ -1112,8 +1114,11 @@ postgresql_opts = [
     ),
     cfg.StrOpt(
         'backup_docker_image',
-        default='openstacktrove/db-backup-postgresql:1.1.2',
+        sample_default='your-registry/your-repo/db-backup-postgresql',
-        help='The docker image used for backup and restore.'
+        help='The docker image used for backup and restore. Trove will uses'
+             'datastore version as the image tag, for example: '
+             'your-registry/your-repo/db-backup-postgresql:12 is used for'
+             'postgresql datastore with version 12'
     ),
     cfg.BoolOpt('icmp', default=False,
                 help='Whether to permit ICMP.',
@@ -1437,8 +1442,11 @@ mariadb_opts = [
     ),
     cfg.StrOpt(
         'backup_docker_image',
-        default='openstacktrove/db-backup-mariadb:1.1.0',
+        sample_default='your-registry/your-repo/db-backup-mariadb',
-        help='The docker image used for backup and restore.'
+        help='The docker image used for backup and restore. Trove will uses'
+             'datastore version as the image tag, for example: '
+             'your-registry/your-repo/db-backup-mariadb:10.3 is used for '
+             'postgresql datastore with version 10.3'
     ),
 ]
 

trove/guestagent/datastore/mysql/service.py

@@ -65,15 +65,18 @@ class MySqlApp(service.BaseMySqlApp):
 
         For example, this method converts openstacktrove/db-backup-mysql:1.0.0
         to openstacktrove/db-backup-mysql5.7:1.0.0
+
+        **deprecated**: this function is for backward compatibility.
         """
         image = cfg.get_configuration_property('backup_docker_image')
-        name, tag = image.rsplit(':', 1)
+        if not self._image_has_tag(image):
-
+            return super().get_backup_image()
-        # Get minor version
+        else:
-        cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
+            name, tag = image.rsplit(':', 1)
-        minor_ver = f"{cur_ver.major}.{cur_ver.minor}"
+            # Get minor version
-
+            cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
-        return f"{name}{minor_ver}:{tag}"
+            minor_ver = f"{cur_ver.major}.{cur_ver.minor}"
+            return f"{name}{minor_ver}:{tag}"
 
     def get_backup_strategy(self):
         """Get backup strategy.

trove/guestagent/datastore/postgres/service.py

@@ -263,8 +263,8 @@ class PgSqlApp(service.BaseDbApp):
     def restore_backup(self, context, backup_info, restore_location):
         backup_id = backup_info['id']
         storage_driver = CONF.storage_strategy
-        backup_driver = cfg.get_configuration_property('backup_strategy')
+        backup_driver = self.get_backup_strategy()
-        image = cfg.get_configuration_property('backup_docker_image')
+        image = self.get_backup_image()
         name = 'db_restore'
         volumes = {
             '/var/lib/postgresql/data': {

trove/guestagent/datastore/service.py

@@ -415,8 +415,32 @@ class BaseDbApp(object):
         self.reset_configuration(config_contents)
         self.start_db(update_db=True, ds_version=ds_version)
 
+    @staticmethod
+    def _image_has_tag(image):
+        """
+        Whether docker_image being config with tag
+            "example.domain:5000/repo/image_name:tag",
+            "example.domain:5000/repo/image-name:tag",
+            "example.domain:5000/repo/image-name:tag_tag",
+            "example.domain:5000/repo/image_name:tag-tag",
+            "example.domain:5000/repo/image-name",
+            "example.domain:5000/repo/image_name",
+            "example.domain:5000:5000/repo/image-name",
+            "example.domain/repo/image-name",
+            "example.domain/repo/image-name:tag"
+
+        Returns:
+            - True if match
+        """
+        return image.split('/')[-1].find(':') > 0
+
     def get_backup_image(self):
-        return cfg.get_configuration_property('backup_docker_image')
+        image = cfg.get_configuration_property('backup_docker_image')
+        if not self._image_has_tag(image):
+            ds_version = CONF.datastore_version
+            image = (f'{image}:latest' if not ds_version else
+                     f'{image}:{ds_version}')
+        return image
 
     def get_backup_strategy(self):
         return cfg.get_configuration_property('backup_strategy')

trove/tests/unittests/guestagent/datastore/test_service.py

@@ -0,0 +1,88 @@
+# Copyright 2023 BizflyCloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+from unittest import mock
+
+from trove.common import cfg
+from trove.guestagent.datastore.mariadb import service
+from trove.guestagent.datastore.mysql import service as mysql_service
+from trove.guestagent.datastore import service as base_service
+from trove.tests.unittests import trove_testtools
+
+
+CONF = cfg.CONF
+
+
+class TestService(trove_testtools.TestCase):
+    def setUp(self):
+        super(TestService, self).setUp()
+        _docker_client = mock.MagicMock()
+        status = mock.MagicMock()
+        self.app = service.MariaDBApp(_docker_client, status)
+        self.mysql_app = mysql_service.MySqlApp(_docker_client, status)
+
+    def test_get_backup_image_with_tag(self):
+        self.patch_datastore_manager('mariadb')
+        CONF.set_override('backup_docker_image',
+                          'example.domain/repo/mariadb:tag', 'mariadb')
+        image = self.app.get_backup_image()
+        self.assertEqual(CONF.mariadb.backup_docker_image, image)
+
+    def test_get_backup_image_without_tag(self):
+        self.patch_datastore_manager('mariadb')
+        CONF.set_override('backup_docker_image',
+                          'example.domain/repo/mariadb', 'mariadb')
+        self.patch_conf_property('datastore_version', '10.4')
+        image = self.app.get_backup_image()
+        _img = f'{CONF.mariadb.backup_docker_image}:{CONF.datastore_version}'
+        self.assertEqual(_img, image)
+
+    def test_mysql_backup_image_with_tag(self):
+        self.patch_datastore_manager('mysql')
+        CONF.set_override('backup_docker_image',
+                          'example.domain/repo/mysql:1.1.0', 'mysql')
+        self.patch_conf_property('datastore_version', '5.7')
+        image = self.mysql_app.get_backup_image()
+        self.assertEqual(image, "example.domain/repo/mysql5.7:1.1.0")
+
+    def test_mysql_backup_image_without_tag(self):
+        self.patch_datastore_manager('mysql')
+        CONF.set_override('backup_docker_image',
+                          'example.domain/repo/mysql', 'mysql')
+        self.patch_conf_property('datastore_version', '5.7')
+        image = self.mysql_app.get_backup_image()
+        self.assertEqual(image, "example.domain/repo/mysql:5.7")
+
+    def test_image_has_tag(self):
+        fake_values = [
+            "example.domain:5000/repo/image_name:tag",
+            "example.domain:5000/repo/image-name:tag_tag",
+            "example.domain:5000/repo/image_name:tag-tag",
+            "example.domain:5000/repo/image-name",
+            "example.domain:5000/repo/image_name",
+            "example.domain/repo/image-name",
+            "example.domain/repo/image-name:tag"]
+        self.assertTrue(
+            base_service.BaseDbApp._image_has_tag(fake_values[0]))
+        self.assertTrue(
+            base_service.BaseDbApp._image_has_tag(fake_values[1]))
+        self.assertTrue(
+            base_service.BaseDbApp._image_has_tag(fake_values[2]))
+        self.assertFalse(
+            base_service.BaseDbApp._image_has_tag(fake_values[3]))
+        self.assertFalse(
+            base_service.BaseDbApp._image_has_tag(fake_values[4]))
+        self.assertFalse(
+            base_service.BaseDbApp._image_has_tag(fake_values[5]))
+        self.assertTrue(
+            base_service.BaseDbApp._image_has_tag(fake_values[6]))