openstack/watcher
Code Issues Proposed changes

Updated purge to now include goals and strategies

Browse Source 
In this changeset, I updated the purge script to now take into
account the registered goals and strategies.

Partially Implements: blueprint get-goal-from-strategy

Change-Id: I2f1d58bb812fa45bc4bc6467760a071d8612e6a4
This commit is contained in:
Vincent Françoise 2016-04-12 09:41:41 +02:00
parent 18e5c7d844
commit ff611544fb
2 changed files with 223 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
watcher/db/purge.py
View File

@ -47,6 +47,8 @@ class WatcherObjectsMap(object):
# This is for generating the .pot translations
keymap = collections.OrderedDict([
("goals", _("Goals")),
("strategies", _("Strategies")),
("audit_templates", _("Audit Templates")),
("audits", _("Audits")),
("action_plans", _("Action Plans")),
@ -143,9 +145,9 @@ class PurgeCommand(object):
query_func = None
if not utils.is_uuid_like(uuid_or_name):
query_func = objects.audit_template.AuditTemplate.get_by_name
query_func = objects.AuditTemplate.get_by_name
else:
query_func = objects.audit_template.AuditTemplate.get_by_uuid
query_func = objects.AuditTemplate.get_by_uuid
try:
audit_template = query_func(cls.ctx, uuid_or_name)
@ -159,45 +161,64 @@ class PurgeCommand(object):
return audit_template.uuid
def _find_goals(self, filters=None):
return objects.Goal.list(self.ctx, filters=filters)
def _find_strategies(self, filters=None):
return objects.Strategy.list(self.ctx, filters=filters)
def _find_audit_templates(self, filters=None):
return objects.audit_template.AuditTemplate.list(
self.ctx, filters=filters)
return objects.AuditTemplate.list(self.ctx, filters=filters)
def _find_audits(self, filters=None):
return objects.audit.Audit.list(self.ctx, filters=filters)
return objects.Audit.list(self.ctx, filters=filters)
def _find_action_plans(self, filters=None):
return objects.action_plan.ActionPlan.list(self.ctx, filters=filters)
return objects.ActionPlan.list(self.ctx, filters=filters)
def _find_actions(self, filters=None):
return objects.action.Action.list(self.ctx, filters=filters)
return objects.Action.list(self.ctx, filters=filters)
def _find_orphans(self):
orphans = WatcherObjectsMap()
filters = dict(deleted=False)
audit_templates = objects.audit_template.AuditTemplate.list(
self.ctx, filters=filters)
audits = objects.audit.Audit.list(self.ctx, filters=filters)
action_plans = objects.action_plan.ActionPlan.list(
self.ctx, filters=filters)
actions = objects.action.Action.list(self.ctx, filters=filters)
goals = objects.Goal.list(self.ctx, filters=filters)
strategies = objects.Strategy.list(self.ctx, filters=filters)
audit_templates = objects.AuditTemplate.list(self.ctx, filters=filters)
audits = objects.Audit.list(self.ctx, filters=filters)
action_plans = objects.ActionPlan.list(self.ctx, filters=filters)
actions = objects.Action.list(self.ctx, filters=filters)
audit_template_ids = set(at.id for at in audit_templates)
goal_ids = set(g.id for g in goals)
orphans.strategies = [
strategy for strategy in strategies
if strategy.goal_id not in goal_ids]
strategy_ids = [s.id for s in (s for s in strategies
if s not in orphans.strategies)]
orphans.audit_templates = [
audit_template for audit_template in audit_templates
if audit_template.goal_id not in goal_ids or
(audit_template.strategy_id and
audit_template.strategy_id not in strategy_ids)]
audit_template_ids = [at.id for at in audit_templates
if at not in orphans.audit_templates]
orphans.audits = [
audit for audit in audits
if audit.audit_template_id not in audit_template_ids]
# Objects with orphan parents are themselves orphans
audit_ids = [audit.id for audit in (a for a in audits
if a not in orphans.audits)]
audit_ids = [audit.id for audit in audits
if audit not in orphans.audits]
orphans.action_plans = [
ap for ap in action_plans
if ap.audit_id not in audit_ids]
# Objects with orphan parents are themselves orphans
action_plan_ids = [ap.id for ap in (a for a in action_plans
if a not in orphans.action_plans)]
action_plan_ids = [ap.id for ap in action_plans
if ap not in orphans.action_plans]
orphans.actions = [
action for action in actions
if action.action_plan_id not in action_plan_ids]
@ -209,18 +230,21 @@ class PurgeCommand(object):
def _find_soft_deleted_objects(self):
to_be_deleted = WatcherObjectsMap()
expiry_date = self.get_expiry_date()
filters = dict(deleted=True)
if self.uuid:
filters["uuid"] = self.uuid
if expiry_date:
filters.update(dict(deleted_at__lt=expiry_date))
to_be_deleted.goals.extend(self._find_goals(filters))
to_be_deleted.strategies.extend(self._find_strategies(filters))
to_be_deleted.audit_templates.extend(
self._find_audit_templates(filters))
to_be_deleted.audits.extend(self._find_audits(filters))
to_be_deleted.action_plans.extend(self._find_action_plans(filters))
to_be_deleted.action_plans.extend(
self._find_action_plans(filters))
to_be_deleted.actions.extend(self._find_actions(filters))
soft_deleted_objs = self._find_related_objects(
@ -233,6 +257,23 @@ class PurgeCommand(object):
def _find_related_objects(self, objects_map, base_filters=None):
base_filters = base_filters or {}
for goal in objects_map.goals:
filters = {}
filters.update(base_filters)
filters.update(dict(goal_id=goal.id))
related_objs = WatcherObjectsMap()
related_objs.strategies = self._find_strategies(filters)
related_objs.audit_templates = self._find_audit_templates(filters)
objects_map += related_objs
for strategy in objects_map.strategies:
filters = {}
filters.update(base_filters)
filters.update(dict(strategy_id=strategy.id))
related_objs = WatcherObjectsMap()
related_objs.audit_templates = self._find_audit_templates(filters)
objects_map += related_objs
for audit_template in objects_map.audit_templates:
filters = {}
filters.update(base_filters)
@ -282,21 +323,48 @@ class PurgeCommand(object):
return self._delete_up_to_max
def _aggregate_objects(self):
"""Objects aggregated on a 'per audit template' basis"""
"""Objects aggregated on a 'per goal' basis"""
# todo: aggregate orphans as well
aggregate = []
for audit_template in self._objects_map.audit_templates:
for goal in self._objects_map.goals:
related_objs = WatcherObjectsMap()
related_objs.audit_templates = [audit_template]
# goals
related_objs.goals = [goal]
# strategies
goal_ids = [goal.id]
related_objs.strategies = [
strategy for strategy in self._objects_map.strategies
if strategy.goal_id in goal_ids
]
# audit templates
strategy_ids = [
strategy.id for strategy in related_objs.strategies]
related_objs.audit_templates = [
at for at in self._objects_map.audit_templates
if at.goal_id in goal_ids or
(at.strategy_id and at.strategy_id in strategy_ids)
]
# audits
audit_template_ids = [
audit_template.id
for audit_template in related_objs.audit_templates]
related_objs.audits = [
audit for audit in self._objects_map.audits
if audit.audit_template_id == audit_template.id
if audit.audit_template_id in audit_template_ids
]
# action plans
audit_ids = [audit.id for audit in related_objs.audits]
related_objs.action_plans = [
action_plan for action_plan in self._objects_map.action_plans
if action_plan.audit_id in audit_ids
]
# actions
action_plan_ids = [
action_plan.id for action_plan in related_objs.action_plans
]

157
watcher/tests/db/test_purge.py
View File

@ -71,6 +71,9 @@ class TestPurgeCommand(base.DbTestCase):
yield seed
seed += 1
def generate_unique_name(self, prefix):
return "%s%s" % (prefix, uuid.uuid4())
def _data_setup(self):
# All the 1's are soft_deleted and are expired
# All the 2's are soft_deleted but are not expired
@ -80,21 +83,62 @@ class TestPurgeCommand(base.DbTestCase):
self.cmd.age_in_days = 10
self.cmd.max_number = None
self.cmd.orphans = True
gen_name = lambda: "Audit Template %s" % uuid.uuid4()
self.audit_template1_name = gen_name()
self.audit_template2_name = gen_name()
self.audit_template3_name = gen_name()
goal1_name = "GOAL_1"
goal2_name = "GOAL_2"
goal3_name = "GOAL_3"
strategy1_name = "strategy_1"
strategy2_name = "strategy_2"
strategy3_name = "strategy_3"
self.audit_template1_name = self.generate_unique_name(
prefix="Audit Template 1 ")
self.audit_template2_name = self.generate_unique_name(
prefix="Audit Template 2 ")
self.audit_template3_name = self.generate_unique_name(
prefix="Audit Template 3 ")
with freezegun.freeze_time(self.expired_date):
self.goal1 = obj_utils.create_test_goal(
self.context, id=self._generate_id(), uuid=None,
name=goal1_name, display_name=goal1_name.lower())
self.goal2 = obj_utils.create_test_goal(
self.context, id=self._generate_id(), uuid=None,
name=goal2_name, display_name=goal2_name.lower())
self.goal3 = obj_utils.create_test_goal(
self.context, id=self._generate_id(), uuid=None,
name=goal3_name, display_name=goal3_name.lower())
self.goal1.soft_delete()
with freezegun.freeze_time(self.expired_date):
self.strategy1 = obj_utils.create_test_strategy(
self.context, id=self._generate_id(), uuid=None,
name=strategy1_name, display_name=strategy1_name.lower(),
goal_id=self.goal1.id)
self.strategy2 = obj_utils.create_test_strategy(
self.context, id=self._generate_id(), uuid=None,
name=strategy2_name, display_name=strategy2_name.lower(),
goal_id=self.goal2.id)
self.strategy3 = obj_utils.create_test_strategy(
self.context, id=self._generate_id(), uuid=None,
name=strategy3_name, display_name=strategy3_name.lower(),
goal_id=self.goal3.id)
self.strategy1.soft_delete()
with freezegun.freeze_time(self.expired_date):
self.audit_template1 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template1_name,
id=self._generate_id(), uuid=None)
id=self._generate_id(), uuid=None, goal_id=self.goal1.id,
strategy_id=self.strategy1.id)
self.audit_template2 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template2_name,
id=self._generate_id(), uuid=None)
id=self._generate_id(), uuid=None, goal_id=self.goal2.id,
strategy_id=self.strategy2.id)
self.audit_template3 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template3_name,
id=self._generate_id(), uuid=None)
id=self._generate_id(), uuid=None, goal_id=self.goal3.id,
strategy_id=self.strategy3.id)
self.audit_template1.soft_delete()
with freezegun.freeze_time(self.expired_date):
@ -135,14 +179,21 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_execute_max_number_exceeded(self, m_destroy_audit_template,
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_execute_max_number_exceeded(self,
m_destroy_goal,
m_destroy_strategy,
m_destroy_audit_template,
m_destroy_audit,
m_destroy_action_plan,
m_destroy_action):
self.cmd.age_in_days = None
self.cmd.max_number = 5
self.cmd.max_number = 10
with freezegun.freeze_time(self.fake_today):
self.goal2.soft_delete()
self.strategy2.soft_delete()
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
@ -151,8 +202,10 @@ class TestPurgeCommand(base.DbTestCase):
self.cmd.execute()
# The 1's and the 2's are purgeable (due to age of day set to 0),
# but max_number = 5, and because of no Db integrity violation, we
# should be able to purge only 4 objects.
# but max_number = 10, and because of no Db integrity violation, we
# should be able to purge only 6 objects.
self.assertEqual(m_destroy_goal.call_count, 1)
self.assertEqual(m_destroy_strategy.call_count, 1)
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_audit.call_count, 1)
self.assertEqual(m_destroy_action_plan.call_count, 1)
@ -164,6 +217,8 @@ class TestPurgeCommand(base.DbTestCase):
with freezegun.freeze_time(self.fake_today):
objects_map = self.cmd.find_objects_to_delete()
self.assertEqual(len(objects_map.goals), 1)
self.assertEqual(len(objects_map.strategies), 1)
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.audits), 1)
self.assertEqual(len(objects_map.action_plans), 1)
@ -171,6 +226,8 @@ class TestPurgeCommand(base.DbTestCase):
def test_find_deleted_and_expired_entries(self):
with freezegun.freeze_time(self.fake_today):
self.goal2.soft_delete()
self.strategy2.soft_delete()
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
@ -179,6 +236,8 @@ class TestPurgeCommand(base.DbTestCase):
objects_map = self.cmd.find_objects_to_delete()
# The 1's are purgeable (due to age of day set to 10)
self.assertEqual(len(objects_map.goals), 1)
self.assertEqual(len(objects_map.strategies), 1)
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.audits), 1)
self.assertEqual(len(objects_map.action_plans), 1)
@ -186,9 +245,13 @@ class TestPurgeCommand(base.DbTestCase):
def test_find_deleted_and_nonexpired_related_entries(self):
with freezegun.freeze_time(self.fake_today):
# orphan audit
# orphan audit template
audit_template4 = obj_utils.create_test_audit_template(
self.context, goal_id=404, # Does not exist
name=self.generate_unique_name(prefix="Audit Template 4 "),
strategy_id=None, id=self._generate_id(), uuid=None)
audit4 = obj_utils.create_test_audit(
self.context, audit_template_id=404, # Does not exist
self.context, audit_template_id=audit_template4.id,
id=self._generate_id(), uuid=None)
action_plan4 = obj_utils.create_test_action_plan(
self.context, audit_id=audit4.id,
@ -197,8 +260,12 @@ class TestPurgeCommand(base.DbTestCase):
self.context, action_plan_id=action_plan4.id,
id=self._generate_id(), uuid=None)
audit_template5 = obj_utils.create_test_audit_template(
self.context, goal_id=self.goal1.id,
name=self.generate_unique_name(prefix="Audit Template 5 "),
strategy_id=None, id=self._generate_id(), uuid=None)
audit5 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template1.id,
self.context, audit_template_id=audit_template5.id,
id=self._generate_id(), uuid=None)
action_plan5 = obj_utils.create_test_action_plan(
self.context, audit_id=audit5.id,
@ -207,6 +274,8 @@ class TestPurgeCommand(base.DbTestCase):
self.context, action_plan_id=action_plan5.id,
id=self._generate_id(), uuid=None)
self.goal2.soft_delete()
self.strategy2.soft_delete()
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
@ -216,13 +285,16 @@ class TestPurgeCommand(base.DbTestCase):
# All the 5's should be purged as well even though they are not
# expired because their related audit template is itself expired
audit_template5.soft_delete()
audit5.soft_delete()
action_plan5.soft_delete()
with freezegun.freeze_time(self.fake_today):
objects_map = self.cmd.find_objects_to_delete()
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.goals), 1)
self.assertEqual(len(objects_map.strategies), 1)
self.assertEqual(len(objects_map.audit_templates), 3)
self.assertEqual(len(objects_map.audits), 3)
self.assertEqual(len(objects_map.action_plans), 3)
self.assertEqual(len(objects_map.actions), 3)
@ -234,9 +306,11 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command(self, m_destroy_audit_template,
m_destroy_audit, m_destroy_action_plan,
m_destroy_action):
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_purge_command(self, m_destroy_goal, m_destroy_strategy,
m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
@ -253,13 +327,20 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_purge_command_with_nonexpired_related_entries(
self, m_destroy_audit_template, m_destroy_audit,
self, m_destroy_goal, m_destroy_strategy,
m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
with freezegun.freeze_time(self.fake_today):
# orphan audit
# orphan audit template
audit_template4 = obj_utils.create_test_audit_template(
self.context, goal_id=404, # Does not exist
name=self.generate_unique_name(prefix="Audit Template 4 "),
strategy_id=None, id=self._generate_id(), uuid=None)
audit4 = obj_utils.create_test_audit(
self.context, audit_template_id=404, # Does not exist
self.context, audit_template_id=audit_template4.id,
id=self._generate_id(), uuid=None)
action_plan4 = obj_utils.create_test_action_plan(
self.context, audit_id=audit4.id,
@ -268,8 +349,12 @@ class TestPurgeCommand(base.DbTestCase):
self.context, action_plan_id=action_plan4.id,
id=self._generate_id(), uuid=None)
audit_template5 = obj_utils.create_test_audit_template(
self.context, goal_id=self.goal1.id,
name=self.generate_unique_name(prefix="Audit Template 5 "),
strategy_id=None, id=self._generate_id(), uuid=None)
audit5 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template1.id,
self.context, audit_template_id=audit_template5.id,
id=self._generate_id(), uuid=None)
action_plan5 = obj_utils.create_test_action_plan(
self.context, audit_id=audit5.id,
@ -278,6 +363,8 @@ class TestPurgeCommand(base.DbTestCase):
self.context, action_plan_id=action_plan5.id,
id=self._generate_id(), uuid=None)
self.goal2.soft_delete()
self.strategy2.soft_delete()
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
@ -287,13 +374,16 @@ class TestPurgeCommand(base.DbTestCase):
# All the 5's should be purged as well even though they are not
# expired because their related audit template is itself expired
audit_template5.soft_delete()
audit5.soft_delete()
action_plan5.soft_delete()
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_goal.call_count, 1)
self.assertEqual(m_destroy_strategy.call_count, 1)
self.assertEqual(m_destroy_audit_template.call_count, 3)
self.assertEqual(m_destroy_audit.call_count, 3)
self.assertEqual(m_destroy_action_plan.call_count, 3)
self.assertEqual(m_destroy_action.call_count, 3)
@ -312,8 +402,11 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_purge_command_with_audit_template_ok(
self, m_destroy_audit_template, m_destroy_audit,
self, m_destroy_goal, m_destroy_strategy,
m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template1.uuid
@ -321,6 +414,8 @@ class TestPurgeCommand(base.DbTestCase):
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_goal.call_count, 0)
self.assertEqual(m_destroy_strategy.call_count, 0)
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_audit.call_count, 1)
self.assertEqual(m_destroy_action_plan.call_count, 1)
@ -339,8 +434,11 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_purge_command_with_audit_template_not_expired(
self, m_destroy_audit_template, m_destroy_audit,
self, m_destroy_goal, m_destroy_strategy,
m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template2.uuid
@ -348,6 +446,8 @@ class TestPurgeCommand(base.DbTestCase):
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_goal.call_count, 0)
self.assertEqual(m_destroy_strategy.call_count, 0)
self.assertEqual(m_destroy_audit_template.call_count, 0)
self.assertEqual(m_destroy_audit.call_count, 0)
self.assertEqual(m_destroy_action_plan.call_count, 0)
@ -357,8 +457,11 @@ class TestPurgeCommand(base.DbTestCase):
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
@mock.patch.object(dbapi.Connection, "destroy_strategy")
@mock.patch.object(dbapi.Connection, "destroy_goal")
def test_purge_command_with_audit_template_not_soft_deleted(
self, m_destroy_audit_template, m_destroy_audit,
self, m_destroy_goal, m_destroy_strategy,
m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template3.uuid
@ -366,6 +469,8 @@ class TestPurgeCommand(base.DbTestCase):
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_goal.call_count, 0)
self.assertEqual(m_destroy_strategy.call_count, 0)
self.assertEqual(m_destroy_audit_template.call_count, 0)
self.assertEqual(m_destroy_audit.call_count, 0)
self.assertEqual(m_destroy_action_plan.call_count, 0)