diff --git a/doc/source/overview_reaper.rst b/doc/source/overview_reaper.rst index 0488a92863..d42539c6d6 100644 --- a/doc/source/overview_reaper.rst +++ b/doc/source/overview_reaper.rst @@ -40,6 +40,12 @@ troublesome spot. The account reaper will keep trying to delete an account until it eventually becomes empty, at which point the database reclaim process within the db_replicator will eventually remove the database files. +Sometimes a persistent error state can prevent some object or container +from being deleted. If this happens, you will see a message such as "Account + has not been reaped since " in the log. You can control when +this is logged with the reap_warn_after value in the [account-reaper] section +of the account-server.conf file. By default this is 30 days. + ------- History ------- diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 9277616274..6d981649c1 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -117,3 +117,11 @@ use = egg:swift#recon # immediately; you can set this to delay its work however. The value is in # seconds; 2592000 = 30 days for example. # delay_reaping = 0 +# If the account fails to be be reaped due to a persistent error, the +# account reaper will log a message such as: +# Account has not been reaped since +# You can search logs for this message if space is not being reclaimed +# after you delete account(s). +# Default is 2592000 seconds (30 days). This is in addition to any time +# requested by delay_reaping. +# reap_warn_after = 2592000 diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 385d95275e..790b8663b0 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -17,7 +17,7 @@ import os import random from logging import DEBUG from math import sqrt -from time import time +from time import time, ctime from eventlet import GreenPool, sleep, Timeout @@ -72,6 +72,8 @@ class AccountReaper(Daemon): swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.delay_reaping = int(conf.get('delay_reaping') or 0) + reap_warn_after = float(conf.get('reap_warn_after') or 86400 * 30) + self.reap_not_done_after = reap_warn_after + self.delay_reaping def get_account_ring(self): """ The account :class:`swift.common.ring.Ring` for the cluster. """ @@ -240,6 +242,8 @@ class AccountReaper(Daemon): self.logger.exception( _('Exception with containers for account %s'), account) marker = containers[-1][0] + if marker == '': + break log = 'Completed pass on account %s' % account except (Exception, Timeout): self.logger.exception( @@ -268,6 +272,10 @@ class AccountReaper(Daemon): log += _(', elapsed: %.02fs') % (time() - begin) self.logger.info(log) self.logger.timing_since('timing', self.start_time) + if self.stats_containers_remaining and \ + begin - float(info['delete_timestamp']) >= self.reap_not_done_after: + self.logger.warn(_('Account %s has not been reaped since %s') % + (account, ctime(float(info['delete_timestamp'])))) return True def reap_container(self, account, account_partition, account_nodes, @@ -346,6 +354,8 @@ class AccountReaper(Daemon): {'container': container, 'account': account}) marker = objects[-1]['name'] + if marker == '': + break successes = 0 failures = 0 for node in nodes: diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index 5ddaf2ac03..858dadf4af 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -46,6 +46,15 @@ class TestReaper(unittest.TestCase): self.assertRaises(ValueError, reaper.AccountReaper, {'delay_reaping': 'abc'}) + def test_reap_warn_after_conf_set(self): + conf = {'delay_reaping': '2', 'reap_warn_after': '3'} + r = reaper.AccountReaper(conf) + self.assertEquals(r.reap_not_done_after, 5) + + def test_reap_warn_after_conf_bad_value(self): + self.assertRaises(ValueError, reaper.AccountReaper, + {'reap_warn_after': 'abc'}) + def test_reap_delay(self): time_value = [100]