diff --git a/cinder/tests/unit/volume/drivers/test_rbd.py b/cinder/tests/unit/volume/drivers/test_rbd.py index fdab2d1370e..bbbcc2e06c4 100644 --- a/cinder/tests/unit/volume/drivers/test_rbd.py +++ b/cinder/tests/unit/volume/drivers/test_rbd.py @@ -21,10 +21,12 @@ import uuid import castellan import ddt +import errno import mock from mock import call from oslo_utils import imageutils from oslo_utils import units +import time from cinder import context from cinder import db @@ -73,6 +75,11 @@ class MockOSErrorException(MockException): """Used as mock for rbd.OSError.""" +class MockPermissionError(MockException): + """Used as mock for PermissionError.""" + errno = errno.EPERM + + class KeyObject(object): def get_encoded(arg): return "asdf".encode('utf-8') @@ -109,6 +116,7 @@ def common_mocks(f): inst.mock_rbd.ImageNotFound = MockImageNotFoundException inst.mock_rbd.ImageExists = MockImageExistsException inst.mock_rbd.InvalidArgument = MockImageNotFoundException + inst.mock_rbd.PermissionError = MockPermissionError inst.driver.rbd = inst.mock_rbd inst.driver.rados = inst.mock_rados @@ -190,6 +198,7 @@ class RBDTestCase(test.TestCase): self.cfg.rados_connection_retries = 3 self.cfg.rados_connection_interval = 5 self.cfg.backup_use_temp_snapshot = False + self.cfg.enable_deferred_deletion = False mock_exec = mock.Mock() mock_exec.return_value = ('', '') @@ -659,6 +668,112 @@ class RBDTestCase(test.TestCase): self.assertEqual( 1, self.driver.rbd.RBD.return_value.remove.call_count) + @common_mocks + def test_deferred_deletion(self): + client = self.mock_client.return_value + + self.driver.rbd.Image.return_value.list_snaps.return_value = [] + + with mock.patch.object(self.driver, '_get_clone_info') as \ + mock_get_clone_info: + with mock.patch.object(self.driver, '_delete_backup_snaps') as \ + mock_delete_backup_snaps: + mock_get_clone_info.return_value = (None, None, None) + self.cfg.enable_deferred_deletion = True + self.cfg.deferred_deletion_delay = 0 + + self.driver.delete_volume(self.volume_a) + + mock_get_clone_info.assert_called_once_with( + self.mock_rbd.Image.return_value, + self.volume_a.name, + None) + (self.driver.rbd.Image.return_value + .list_snaps.assert_called_once_with()) + client.__enter__.assert_called_once_with() + client.__exit__.assert_called_once_with(None, None, None) + mock_delete_backup_snaps.assert_called_once_with( + self.mock_rbd.Image.return_value) + self.assertFalse( + self.driver.rbd.Image.return_value.unprotect_snap.called) + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_move.call_count) + + @common_mocks + def test_deferred_deletion_periodic_task(self): + self.cfg.rados_connect_timeout = -1 + self.cfg.enable_deferred_deletion = True + self.cfg.deferred_deletion_purge_interval = 1 + + self.driver._start_periodic_tasks() + + time.sleep(1) + self.assertTrue(self.driver.rbd.RBD.return_value.trash_list.called) + self.assertFalse(self.driver.rbd.RBD.return_value.trash_remove.called) + + @common_mocks + def test_deferred_deletion_trash_purge(self): + with mock.patch.object(self.driver.rbd.RBD(), 'trash_list') as \ + mock_trash_list: + mock_trash_list.return_value = [self.volume_a] + self.cfg.enable_deferred_deletion = True + + self.driver._trash_purge() + + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_list.call_count) + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_remove.call_count) + + @common_mocks + def test_deferred_deletion_trash_purge_not_expired(self): + with mock.patch.object(self.driver.rbd.RBD(), 'trash_list') as \ + mock_trash_list: + mock_trash_list.return_value = [self.volume_a] + self.mock_rbd.RBD.return_value.trash_remove.side_effect = ( + self.mock_rbd.PermissionError) + self.cfg.enable_deferred_deletion = True + + self.driver._trash_purge() + + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_list.call_count) + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_remove.call_count) + # Make sure the exception was raised + self.assertEqual(1, len(RAISED_EXCEPTIONS)) + self.assertIn(self.mock_rbd.PermissionError, RAISED_EXCEPTIONS) + + @common_mocks + def test_deferred_deletion_w_parent(self): + _get_clone_info_return_values = [ + (None, self.volume_b.name, None), + (None, None, None)] + with mock.patch.object(self.driver, '_get_clone_info', + side_effect = _get_clone_info_return_values): + self.cfg.enable_deferred_deletion = True + self.cfg.deferred_deletion_delay = 0 + + self.driver.delete_volume(self.volume_a) + + self.assertEqual( + 1, self.driver.rbd.RBD.return_value.trash_move.call_count) + + @common_mocks + def test_deferred_deletion_w_deleted_parent(self): + _get_clone_info_return_values = [ + (None, "%s.deleted" % self.volume_b.name, None), + (None, None, None)] + with mock.patch.object(self.driver, '_get_clone_info', + side_effect = _get_clone_info_return_values): + self.cfg.enable_deferred_deletion = True + self.cfg.deferred_deletion_delay = 0 + + self.driver.delete_volume(self.volume_a) + + self.assertEqual( + 2, self.driver.rbd.RBD.return_value.trash_move.call_count) + @common_mocks def delete_volume_not_found(self): self.mock_rbd.Image.side_effect = self.mock_rbd.ImageNotFound diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py index 2b66d5a3204..8075df8cce2 100644 --- a/cinder/volume/drivers/rbd.py +++ b/cinder/volume/drivers/rbd.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import binascii +import errno import json import math import os @@ -25,6 +26,7 @@ from eventlet import tpool from os_brick.initiator import linuxrbd from oslo_config import cfg from oslo_log import log as logging +from oslo_service import loopingcall from oslo_utils import encodeutils from oslo_utils import excutils from oslo_utils import fileutils @@ -111,6 +113,17 @@ RBD_OPTS = [ "Cinder core code for allocated_capacity_gb. This " "reduces the load on the Ceph cluster as well as on the " "volume service."), + cfg.BoolOpt('enable_deferred_deletion', default=False, + help='Enable deferred deletion. Upon deletion, volumes are ' + 'tagged for deletion but will only be removed ' + 'asynchronously at a later time.'), + cfg.IntOpt('deferred_deletion_delay', default=0, + help='Time delay in seconds before a volume is eligible ' + 'for permanent removal after being tagged for deferred ' + 'deletion.'), + cfg.IntOpt('deferred_deletion_purge_interval', default=60, + help='Number of seconds between runs of the periodic task' + 'to purge volumes tagged for deletion.'), ] CONF = cfg.CONF @@ -280,6 +293,42 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, remote = self._active_config return (remote.get('name'), remote.get('conf'), remote.get('user')) + def _trash_purge(self): + LOG.info("Purging trash for backend '%s'", self._backend_name) + with RADOSClient(self) as client: + for vol in self.RBDProxy().trash_list(client.ioctx): + try: + self.RBDProxy().trash_remove(client.ioctx, vol.get('id')) + LOG.info("Deleted %s from trash for backend '%s'", + vol.get('name'), + self._backend_name) + except Exception as e: + # NOTE(arne_wiebalck): trash_remove raises EPERM in case + # the volume's deferral time has not expired yet, so we + # want to explicitly handle this "normal" situation. + # All other exceptions, e.g. ImageBusy, are not re-raised + # so that the periodic purge retries on the next iteration + # and leaves ERRORs in the logs in case the deletion fails + # repeatedly. + if e.errno == errno.EPERM: + LOG.debug("%s has not expired yet on backend '%s'", + vol.get('name'), + self._backend_name) + else: + LOG.exception("Error deleting %s from trash " + "backend '%s'", + vol.get('name'), + self._backend_name) + + def _start_periodic_tasks(self): + if self.configuration.enable_deferred_deletion: + LOG.info("Starting periodic trash purge for backend '%s'", + self._backend_name) + deferred_deletion_ptask = loopingcall.FixedIntervalLoopingCall( + self._trash_purge) + deferred_deletion_ptask.start( + interval=self.configuration.deferred_deletion_purge_interval) + def check_for_setup_error(self): """Returns an error if prerequisites aren't met.""" if rados is None: @@ -297,6 +346,18 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, with RADOSClient(self): pass + # NOTE(arne_wiebalck): If deferred deletion is enabled, check if the + # local Ceph client has support for the trash API. + if self.configuration.enable_deferred_deletion: + if not hasattr(self.RBDProxy(), 'trash_list'): + msg = _("Deferred deletion is enabled, but the local Ceph " + "client has no support for the trash API. Support " + "for this feature started with v12.2.0 Luminous.") + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + + self._start_periodic_tasks() + def RBDProxy(self): return tpool.Proxy(self.rbd.RBD()) @@ -917,7 +978,14 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, # keep walking up the chain if it is itself a clone. if (not parent_has_snaps) and parent_name.endswith('.deleted'): LOG.debug("deleting parent %s", parent_name) - self.RBDProxy().remove(client.ioctx, parent_name) + if self.configuration.enable_deferred_deletion: + LOG.debug("moving volume %s to trash", parent_name) + delay = self.configuration.deferred_deletion_delay + self.RBDProxy().trash_move(client.ioctx, + parent_name, + delay) + else: + self.RBDProxy().remove(client.ioctx, parent_name) # Now move up to grandparent if there is one if g_parent: @@ -967,7 +1035,14 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, self.configuration.rados_connection_interval, self.configuration.rados_connection_retries) def _try_remove_volume(client, volume_name): - self.RBDProxy().remove(client.ioctx, volume_name) + if self.configuration.enable_deferred_deletion: + LOG.debug("moving volume %s to trash", volume_name) + delay = self.configuration.deferred_deletion_delay + self.RBDProxy().trash_move(client.ioctx, + volume_name, + delay) + else: + self.RBDProxy().remove(client.ioctx, volume_name) if clone_snap is None: LOG.debug("deleting rbd volume %s", volume_name) diff --git a/releasenotes/notes/support_deferred_deletion_in_RBD-0c5d96f8eac5b50a.yaml b/releasenotes/notes/support_deferred_deletion_in_RBD-0c5d96f8eac5b50a.yaml new file mode 100644 index 00000000000..7278ce06ef4 --- /dev/null +++ b/releasenotes/notes/support_deferred_deletion_in_RBD-0c5d96f8eac5b50a.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for deferred deletion in the RBD volume driver.