Merge "Add db purge command"
This commit is contained in:
commit
585206940a
glance
@ -29,6 +29,7 @@ from __future__ import print_function
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
# If ../glance/__init__.py exists, add ../ to Python search path, so that
|
# If ../glance/__init__.py exists, add ../ to Python search path, so that
|
||||||
# it will override what happens to be installed in /usr/(local/)lib/python...
|
# it will override what happens to be installed in /usr/(local/)lib/python...
|
||||||
@ -46,6 +47,7 @@ import six
|
|||||||
|
|
||||||
from glance.common import config
|
from glance.common import config
|
||||||
from glance.common import exception
|
from glance.common import exception
|
||||||
|
from glance import context
|
||||||
from glance.db import migration as db_migration
|
from glance.db import migration as db_migration
|
||||||
from glance.db.sqlalchemy import api as db_api
|
from glance.db.sqlalchemy import api as db_api
|
||||||
from glance.db.sqlalchemy import metadata
|
from glance.db.sqlalchemy import metadata
|
||||||
@ -145,6 +147,26 @@ class DbCommands(object):
|
|||||||
metadata.db_export_metadefs(db_api.get_engine(),
|
metadata.db_export_metadefs(db_api.get_engine(),
|
||||||
path)
|
path)
|
||||||
|
|
||||||
|
@args('--age_in_days', type=int,
|
||||||
|
help='Purge deleted rows older than age in days')
|
||||||
|
@args('--max_rows', type=int,
|
||||||
|
help='Limit number of records to delete')
|
||||||
|
def purge(self, age_in_days=30, max_rows=100):
|
||||||
|
"""Purge deleted rows older than a given age from glance tables."""
|
||||||
|
age_in_days = int(age_in_days)
|
||||||
|
max_rows = int(max_rows)
|
||||||
|
if age_in_days <= 0:
|
||||||
|
print(_("Must supply a positive, non-zero value for age."))
|
||||||
|
exit(1)
|
||||||
|
if age_in_days >= (int(time.time()) / 86400):
|
||||||
|
print(_("Maximal age is count of days since epoch."))
|
||||||
|
exit(1)
|
||||||
|
if max_rows < 1:
|
||||||
|
print(_("Minimal rows limit is 1."))
|
||||||
|
exit(1)
|
||||||
|
ctx = context.get_admin_context(show_deleted=True)
|
||||||
|
db_api.purge_deleted_rows(ctx, age_in_days, max_rows)
|
||||||
|
|
||||||
|
|
||||||
class DbLegacyCommands(object):
|
class DbLegacyCommands(object):
|
||||||
"""Class for managing the db using legacy commands"""
|
"""Class for managing the db using legacy commands"""
|
||||||
|
@ -58,3 +58,12 @@ class RequestContext(context.RequestContext):
|
|||||||
def can_see_deleted(self):
|
def can_see_deleted(self):
|
||||||
"""Admins can see deleted by default"""
|
"""Admins can see deleted by default"""
|
||||||
return self.show_deleted or self.is_admin
|
return self.show_deleted or self.is_admin
|
||||||
|
|
||||||
|
|
||||||
|
def get_admin_context(show_deleted=False):
|
||||||
|
"""Create an administrator context."""
|
||||||
|
return RequestContext(auth_token=None,
|
||||||
|
tenant=None,
|
||||||
|
is_admin=True,
|
||||||
|
show_deleted=show_deleted,
|
||||||
|
overwrite=False)
|
||||||
|
@ -21,6 +21,7 @@
|
|||||||
|
|
||||||
"""Defines interface for DB access."""
|
"""Defines interface for DB access."""
|
||||||
|
|
||||||
|
import datetime
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
@ -33,6 +34,7 @@ import six
|
|||||||
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
|
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
|
||||||
from six.moves import range
|
from six.moves import range
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
|
from sqlalchemy import MetaData, Table, select
|
||||||
import sqlalchemy.orm as sa_orm
|
import sqlalchemy.orm as sa_orm
|
||||||
import sqlalchemy.sql as sa_sql
|
import sqlalchemy.sql as sa_sql
|
||||||
|
|
||||||
@ -50,7 +52,7 @@ from glance.db.sqlalchemy.metadef_api import object as metadef_object_api
|
|||||||
from glance.db.sqlalchemy.metadef_api import property as metadef_property_api
|
from glance.db.sqlalchemy.metadef_api import property as metadef_property_api
|
||||||
from glance.db.sqlalchemy.metadef_api import tag as metadef_tag_api
|
from glance.db.sqlalchemy.metadef_api import tag as metadef_tag_api
|
||||||
from glance.db.sqlalchemy import models
|
from glance.db.sqlalchemy import models
|
||||||
from glance.i18n import _, _LW
|
from glance.i18n import _, _LW, _LE, _LI
|
||||||
|
|
||||||
BASE = models.BASE
|
BASE = models.BASE
|
||||||
sa_logger = None
|
sa_logger = None
|
||||||
@ -1226,6 +1228,69 @@ def image_tag_get_all(context, image_id, session=None):
|
|||||||
return [tag[0] for tag in tags]
|
return [tag[0] for tag in tags]
|
||||||
|
|
||||||
|
|
||||||
|
def purge_deleted_rows(context, age_in_days, max_rows, session=None):
|
||||||
|
"""Purges soft deleted rows
|
||||||
|
|
||||||
|
Deletes rows of table images, table tasks and all dependent tables
|
||||||
|
according to given age for relevant models.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
age_in_days = int(age_in_days)
|
||||||
|
except ValueError:
|
||||||
|
LOG.exception(_LE('Invalid value for age, %(age)d'),
|
||||||
|
{'age': age_in_days})
|
||||||
|
raise exception.InvalidParameterValue(value=age_in_days,
|
||||||
|
param='age_in_days')
|
||||||
|
try:
|
||||||
|
max_rows = int(max_rows)
|
||||||
|
except ValueError:
|
||||||
|
LOG.exception(_LE('Invalid value for max_rows, %(max_rows)d'),
|
||||||
|
{'max_rows': max_rows})
|
||||||
|
raise exception.InvalidParameterValue(value=max_rows,
|
||||||
|
param='max_rows')
|
||||||
|
|
||||||
|
session = session or get_session()
|
||||||
|
metadata = MetaData(get_engine())
|
||||||
|
deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days)
|
||||||
|
|
||||||
|
tables = []
|
||||||
|
for model_class in models.__dict__.values():
|
||||||
|
if not hasattr(model_class, '__tablename__'):
|
||||||
|
continue
|
||||||
|
if hasattr(model_class, 'deleted'):
|
||||||
|
tables.append(model_class.__tablename__)
|
||||||
|
# get rid of FX constraints
|
||||||
|
for tbl in ('images', 'tasks'):
|
||||||
|
try:
|
||||||
|
tables.remove(tbl)
|
||||||
|
except ValueError:
|
||||||
|
LOG.warning(_LW('Expected table %(tbl)s was not found in DB.'),
|
||||||
|
**locals())
|
||||||
|
else:
|
||||||
|
tables.append(tbl)
|
||||||
|
|
||||||
|
for tbl in tables:
|
||||||
|
tab = Table(tbl, metadata, autoload=True)
|
||||||
|
LOG.info(
|
||||||
|
_LI('Purging deleted rows older than %(age_in_days)d day(s) '
|
||||||
|
'from table %(tbl)s'),
|
||||||
|
**locals()
|
||||||
|
)
|
||||||
|
with session.begin():
|
||||||
|
result = session.execute(
|
||||||
|
tab.delete().where(
|
||||||
|
tab.columns.id.in_(
|
||||||
|
select([tab.columns.id]).where(
|
||||||
|
tab.columns.deleted_at < deleted_age
|
||||||
|
).limit(max_rows)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
rows = result.rowcount
|
||||||
|
LOG.info(_LI('Deleted %(rows)d row(s) from table %(tbl)s'),
|
||||||
|
**locals())
|
||||||
|
|
||||||
|
|
||||||
def user_get_storage_usage(context, owner_id, image_id=None, session=None):
|
def user_get_storage_usage(context, owner_id, image_id=None, session=None):
|
||||||
_check_image_id(image_id)
|
_check_image_id(image_id)
|
||||||
session = session or get_session()
|
session = session or get_session()
|
||||||
@ -1452,7 +1517,8 @@ def _task_get(context, task_id, session=None, force_show_deleted=False):
|
|||||||
|
|
||||||
def _task_update(context, task_ref, values, session=None):
|
def _task_update(context, task_ref, values, session=None):
|
||||||
"""Apply supplied dictionary of values to a task object."""
|
"""Apply supplied dictionary of values to a task object."""
|
||||||
values["deleted"] = False
|
if 'deleted' not in values:
|
||||||
|
values["deleted"] = False
|
||||||
task_ref.update(values)
|
task_ref.update(values)
|
||||||
task_ref.save(session=session)
|
task_ref.save(session=session)
|
||||||
return task_ref
|
return task_ref
|
||||||
|
@ -75,7 +75,7 @@ def build_task_fixture(**kwargs):
|
|||||||
'message': None,
|
'message': None,
|
||||||
'expires_at': None,
|
'expires_at': None,
|
||||||
'created_at': default_datetime,
|
'created_at': default_datetime,
|
||||||
'updated_at': default_datetime
|
'updated_at': default_datetime,
|
||||||
}
|
}
|
||||||
task.update(kwargs)
|
task.update(kwargs)
|
||||||
return task
|
return task
|
||||||
@ -1795,6 +1795,62 @@ class TaskTests(test_utils.BaseTestCase):
|
|||||||
self.assertIsNotNone(del_task['deleted_at'])
|
self.assertIsNotNone(del_task['deleted_at'])
|
||||||
|
|
||||||
|
|
||||||
|
class DBPurgeTests(test_utils.BaseTestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(DBPurgeTests, self).setUp()
|
||||||
|
self.adm_context = context.get_admin_context(show_deleted=True)
|
||||||
|
self.db_api = db_tests.get_db(self.config)
|
||||||
|
db_tests.reset_db(self.db_api)
|
||||||
|
self.image_fixtures, self.task_fixtures = self.build_fixtures()
|
||||||
|
self.create_tasks(self.task_fixtures)
|
||||||
|
self.create_images(self.image_fixtures)
|
||||||
|
|
||||||
|
def build_fixtures(self):
|
||||||
|
dt1 = timeutils.utcnow() - datetime.timedelta(days=5)
|
||||||
|
dt2 = dt1 + datetime.timedelta(days=1)
|
||||||
|
dt3 = dt2 + datetime.timedelta(days=1)
|
||||||
|
fixtures = [
|
||||||
|
{
|
||||||
|
'created_at': dt1,
|
||||||
|
'updated_at': dt1,
|
||||||
|
'deleted_at': dt3,
|
||||||
|
'deleted': True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'created_at': dt1,
|
||||||
|
'updated_at': dt2,
|
||||||
|
'deleted_at': timeutils.utcnow(),
|
||||||
|
'deleted': True,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'created_at': dt2,
|
||||||
|
'updated_at': dt2,
|
||||||
|
'deleted_at': None,
|
||||||
|
'deleted': False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
return (
|
||||||
|
[build_image_fixture(**fixture) for fixture in fixtures],
|
||||||
|
[build_task_fixture(**fixture) for fixture in fixtures],
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_images(self, images):
|
||||||
|
for fixture in images:
|
||||||
|
self.db_api.image_create(self.adm_context, fixture)
|
||||||
|
|
||||||
|
def create_tasks(self, tasks):
|
||||||
|
for fixture in tasks:
|
||||||
|
self.db_api.task_create(self.adm_context, fixture)
|
||||||
|
|
||||||
|
def test_db_purge(self):
|
||||||
|
self.db_api.purge_deleted_rows(self.adm_context, 1, 5)
|
||||||
|
images = self.db_api.image_get_all(self.adm_context)
|
||||||
|
self.assertEqual(len(images), 2)
|
||||||
|
tasks = self.db_api.task_get_all(self.adm_context)
|
||||||
|
self.assertEqual(len(tasks), 2)
|
||||||
|
|
||||||
|
|
||||||
class TestVisibility(test_utils.BaseTestCase):
|
class TestVisibility(test_utils.BaseTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestVisibility, self).setUp()
|
super(TestVisibility, self).setUp()
|
||||||
|
@ -160,6 +160,15 @@ class TestSqlAlchemyQuota(base.DriverQuotaTests,
|
|||||||
self.addCleanup(db_tests.reset)
|
self.addCleanup(db_tests.reset)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDBPurge(base.DBPurgeTests,
|
||||||
|
base.FunctionalInitWrapper):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
db_tests.load(get_db, reset_db)
|
||||||
|
super(TestDBPurge, self).setUp()
|
||||||
|
self.addCleanup(db_tests.reset)
|
||||||
|
|
||||||
|
|
||||||
class TestArtifacts(base_artifacts.ArtifactsTestDriver,
|
class TestArtifacts(base_artifacts.ArtifactsTestDriver,
|
||||||
base_artifacts.ArtifactTests):
|
base_artifacts.ArtifactTests):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
Loading…
x
Reference in New Issue
Block a user