From ff611544fbf35ba49d5db8ca9a165d6d02a61a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Fran=C3=A7oise?= <Vincent.FRANCOISE@b-com.com> Date: Tue, 12 Apr 2016 09:41:41 +0200 Subject: [PATCH] Updated purge to now include goals and strategies In this changeset, I updated the purge script to now take into account the registered goals and strategies. Partially Implements: blueprint get-goal-from-strategy Change-Id: I2f1d58bb812fa45bc4bc6467760a071d8612e6a4 --- watcher/db/purge.py | 116 +++++++++++++++++++----- watcher/tests/db/test_purge.py | 157 +++++++++++++++++++++++++++------ 2 files changed, 223 insertions(+), 50 deletions(-) diff --git a/watcher/db/purge.py b/watcher/db/purge.py index 7241878b8..a66682108 100644 --- a/watcher/db/purge.py +++ b/watcher/db/purge.py @@ -47,6 +47,8 @@ class WatcherObjectsMap(object): # This is for generating the .pot translations keymap = collections.OrderedDict([ + ("goals", _("Goals")), + ("strategies", _("Strategies")), ("audit_templates", _("Audit Templates")), ("audits", _("Audits")), ("action_plans", _("Action Plans")), @@ -143,9 +145,9 @@ class PurgeCommand(object): query_func = None if not utils.is_uuid_like(uuid_or_name): - query_func = objects.audit_template.AuditTemplate.get_by_name + query_func = objects.AuditTemplate.get_by_name else: - query_func = objects.audit_template.AuditTemplate.get_by_uuid + query_func = objects.AuditTemplate.get_by_uuid try: audit_template = query_func(cls.ctx, uuid_or_name) @@ -159,45 +161,64 @@ class PurgeCommand(object): return audit_template.uuid + def _find_goals(self, filters=None): + return objects.Goal.list(self.ctx, filters=filters) + + def _find_strategies(self, filters=None): + return objects.Strategy.list(self.ctx, filters=filters) + def _find_audit_templates(self, filters=None): - return objects.audit_template.AuditTemplate.list( - self.ctx, filters=filters) + return objects.AuditTemplate.list(self.ctx, filters=filters) def _find_audits(self, filters=None): - return objects.audit.Audit.list(self.ctx, filters=filters) + return objects.Audit.list(self.ctx, filters=filters) def _find_action_plans(self, filters=None): - return objects.action_plan.ActionPlan.list(self.ctx, filters=filters) + return objects.ActionPlan.list(self.ctx, filters=filters) def _find_actions(self, filters=None): - return objects.action.Action.list(self.ctx, filters=filters) + return objects.Action.list(self.ctx, filters=filters) def _find_orphans(self): orphans = WatcherObjectsMap() filters = dict(deleted=False) - audit_templates = objects.audit_template.AuditTemplate.list( - self.ctx, filters=filters) - audits = objects.audit.Audit.list(self.ctx, filters=filters) - action_plans = objects.action_plan.ActionPlan.list( - self.ctx, filters=filters) - actions = objects.action.Action.list(self.ctx, filters=filters) + goals = objects.Goal.list(self.ctx, filters=filters) + strategies = objects.Strategy.list(self.ctx, filters=filters) + audit_templates = objects.AuditTemplate.list(self.ctx, filters=filters) + audits = objects.Audit.list(self.ctx, filters=filters) + action_plans = objects.ActionPlan.list(self.ctx, filters=filters) + actions = objects.Action.list(self.ctx, filters=filters) - audit_template_ids = set(at.id for at in audit_templates) + goal_ids = set(g.id for g in goals) + orphans.strategies = [ + strategy for strategy in strategies + if strategy.goal_id not in goal_ids] + + strategy_ids = [s.id for s in (s for s in strategies + if s not in orphans.strategies)] + orphans.audit_templates = [ + audit_template for audit_template in audit_templates + if audit_template.goal_id not in goal_ids or + (audit_template.strategy_id and + audit_template.strategy_id not in strategy_ids)] + + audit_template_ids = [at.id for at in audit_templates + if at not in orphans.audit_templates] orphans.audits = [ audit for audit in audits if audit.audit_template_id not in audit_template_ids] # Objects with orphan parents are themselves orphans - audit_ids = [audit.id for audit in (a for a in audits - if a not in orphans.audits)] + audit_ids = [audit.id for audit in audits + if audit not in orphans.audits] orphans.action_plans = [ ap for ap in action_plans if ap.audit_id not in audit_ids] # Objects with orphan parents are themselves orphans - action_plan_ids = [ap.id for ap in (a for a in action_plans - if a not in orphans.action_plans)] + action_plan_ids = [ap.id for ap in action_plans + if ap not in orphans.action_plans] orphans.actions = [ action for action in actions if action.action_plan_id not in action_plan_ids] @@ -209,18 +230,21 @@ class PurgeCommand(object): def _find_soft_deleted_objects(self): to_be_deleted = WatcherObjectsMap() - expiry_date = self.get_expiry_date() filters = dict(deleted=True) + if self.uuid: filters["uuid"] = self.uuid if expiry_date: filters.update(dict(deleted_at__lt=expiry_date)) + to_be_deleted.goals.extend(self._find_goals(filters)) + to_be_deleted.strategies.extend(self._find_strategies(filters)) to_be_deleted.audit_templates.extend( self._find_audit_templates(filters)) to_be_deleted.audits.extend(self._find_audits(filters)) - to_be_deleted.action_plans.extend(self._find_action_plans(filters)) + to_be_deleted.action_plans.extend( + self._find_action_plans(filters)) to_be_deleted.actions.extend(self._find_actions(filters)) soft_deleted_objs = self._find_related_objects( @@ -233,6 +257,23 @@ class PurgeCommand(object): def _find_related_objects(self, objects_map, base_filters=None): base_filters = base_filters or {} + for goal in objects_map.goals: + filters = {} + filters.update(base_filters) + filters.update(dict(goal_id=goal.id)) + related_objs = WatcherObjectsMap() + related_objs.strategies = self._find_strategies(filters) + related_objs.audit_templates = self._find_audit_templates(filters) + objects_map += related_objs + + for strategy in objects_map.strategies: + filters = {} + filters.update(base_filters) + filters.update(dict(strategy_id=strategy.id)) + related_objs = WatcherObjectsMap() + related_objs.audit_templates = self._find_audit_templates(filters) + objects_map += related_objs + for audit_template in objects_map.audit_templates: filters = {} filters.update(base_filters) @@ -282,21 +323,48 @@ class PurgeCommand(object): return self._delete_up_to_max def _aggregate_objects(self): - """Objects aggregated on a 'per audit template' basis""" + """Objects aggregated on a 'per goal' basis""" # todo: aggregate orphans as well aggregate = [] - for audit_template in self._objects_map.audit_templates: + for goal in self._objects_map.goals: related_objs = WatcherObjectsMap() - related_objs.audit_templates = [audit_template] + + # goals + related_objs.goals = [goal] + + # strategies + goal_ids = [goal.id] + related_objs.strategies = [ + strategy for strategy in self._objects_map.strategies + if strategy.goal_id in goal_ids + ] + + # audit templates + strategy_ids = [ + strategy.id for strategy in related_objs.strategies] + related_objs.audit_templates = [ + at for at in self._objects_map.audit_templates + if at.goal_id in goal_ids or + (at.strategy_id and at.strategy_id in strategy_ids) + ] + + # audits + audit_template_ids = [ + audit_template.id + for audit_template in related_objs.audit_templates] related_objs.audits = [ audit for audit in self._objects_map.audits - if audit.audit_template_id == audit_template.id + if audit.audit_template_id in audit_template_ids ] + + # action plans audit_ids = [audit.id for audit in related_objs.audits] related_objs.action_plans = [ action_plan for action_plan in self._objects_map.action_plans if action_plan.audit_id in audit_ids ] + + # actions action_plan_ids = [ action_plan.id for action_plan in related_objs.action_plans ] diff --git a/watcher/tests/db/test_purge.py b/watcher/tests/db/test_purge.py index 44f7b0d00..5e5d557b8 100644 --- a/watcher/tests/db/test_purge.py +++ b/watcher/tests/db/test_purge.py @@ -71,6 +71,9 @@ class TestPurgeCommand(base.DbTestCase): yield seed seed += 1 + def generate_unique_name(self, prefix): + return "%s%s" % (prefix, uuid.uuid4()) + def _data_setup(self): # All the 1's are soft_deleted and are expired # All the 2's are soft_deleted but are not expired @@ -80,21 +83,62 @@ class TestPurgeCommand(base.DbTestCase): self.cmd.age_in_days = 10 self.cmd.max_number = None self.cmd.orphans = True - gen_name = lambda: "Audit Template %s" % uuid.uuid4() - self.audit_template1_name = gen_name() - self.audit_template2_name = gen_name() - self.audit_template3_name = gen_name() + + goal1_name = "GOAL_1" + goal2_name = "GOAL_2" + goal3_name = "GOAL_3" + + strategy1_name = "strategy_1" + strategy2_name = "strategy_2" + strategy3_name = "strategy_3" + + self.audit_template1_name = self.generate_unique_name( + prefix="Audit Template 1 ") + self.audit_template2_name = self.generate_unique_name( + prefix="Audit Template 2 ") + self.audit_template3_name = self.generate_unique_name( + prefix="Audit Template 3 ") + + with freezegun.freeze_time(self.expired_date): + self.goal1 = obj_utils.create_test_goal( + self.context, id=self._generate_id(), uuid=None, + name=goal1_name, display_name=goal1_name.lower()) + self.goal2 = obj_utils.create_test_goal( + self.context, id=self._generate_id(), uuid=None, + name=goal2_name, display_name=goal2_name.lower()) + self.goal3 = obj_utils.create_test_goal( + self.context, id=self._generate_id(), uuid=None, + name=goal3_name, display_name=goal3_name.lower()) + self.goal1.soft_delete() + + with freezegun.freeze_time(self.expired_date): + self.strategy1 = obj_utils.create_test_strategy( + self.context, id=self._generate_id(), uuid=None, + name=strategy1_name, display_name=strategy1_name.lower(), + goal_id=self.goal1.id) + self.strategy2 = obj_utils.create_test_strategy( + self.context, id=self._generate_id(), uuid=None, + name=strategy2_name, display_name=strategy2_name.lower(), + goal_id=self.goal2.id) + self.strategy3 = obj_utils.create_test_strategy( + self.context, id=self._generate_id(), uuid=None, + name=strategy3_name, display_name=strategy3_name.lower(), + goal_id=self.goal3.id) + self.strategy1.soft_delete() with freezegun.freeze_time(self.expired_date): self.audit_template1 = obj_utils.create_test_audit_template( self.context, name=self.audit_template1_name, - id=self._generate_id(), uuid=None) + id=self._generate_id(), uuid=None, goal_id=self.goal1.id, + strategy_id=self.strategy1.id) self.audit_template2 = obj_utils.create_test_audit_template( self.context, name=self.audit_template2_name, - id=self._generate_id(), uuid=None) + id=self._generate_id(), uuid=None, goal_id=self.goal2.id, + strategy_id=self.strategy2.id) self.audit_template3 = obj_utils.create_test_audit_template( self.context, name=self.audit_template3_name, - id=self._generate_id(), uuid=None) + id=self._generate_id(), uuid=None, goal_id=self.goal3.id, + strategy_id=self.strategy3.id) self.audit_template1.soft_delete() with freezegun.freeze_time(self.expired_date): @@ -135,14 +179,21 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") - def test_execute_max_number_exceeded(self, m_destroy_audit_template, + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") + def test_execute_max_number_exceeded(self, + m_destroy_goal, + m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, m_destroy_action_plan, m_destroy_action): self.cmd.age_in_days = None - self.cmd.max_number = 5 + self.cmd.max_number = 10 with freezegun.freeze_time(self.fake_today): + self.goal2.soft_delete() + self.strategy2.soft_delete() self.audit_template2.soft_delete() self.audit2.soft_delete() self.action_plan2.soft_delete() @@ -151,8 +202,10 @@ class TestPurgeCommand(base.DbTestCase): self.cmd.execute() # The 1's and the 2's are purgeable (due to age of day set to 0), - # but max_number = 5, and because of no Db integrity violation, we - # should be able to purge only 4 objects. + # but max_number = 10, and because of no Db integrity violation, we + # should be able to purge only 6 objects. + self.assertEqual(m_destroy_goal.call_count, 1) + self.assertEqual(m_destroy_strategy.call_count, 1) self.assertEqual(m_destroy_audit_template.call_count, 1) self.assertEqual(m_destroy_audit.call_count, 1) self.assertEqual(m_destroy_action_plan.call_count, 1) @@ -164,6 +217,8 @@ class TestPurgeCommand(base.DbTestCase): with freezegun.freeze_time(self.fake_today): objects_map = self.cmd.find_objects_to_delete() + self.assertEqual(len(objects_map.goals), 1) + self.assertEqual(len(objects_map.strategies), 1) self.assertEqual(len(objects_map.audit_templates), 1) self.assertEqual(len(objects_map.audits), 1) self.assertEqual(len(objects_map.action_plans), 1) @@ -171,6 +226,8 @@ class TestPurgeCommand(base.DbTestCase): def test_find_deleted_and_expired_entries(self): with freezegun.freeze_time(self.fake_today): + self.goal2.soft_delete() + self.strategy2.soft_delete() self.audit_template2.soft_delete() self.audit2.soft_delete() self.action_plan2.soft_delete() @@ -179,6 +236,8 @@ class TestPurgeCommand(base.DbTestCase): objects_map = self.cmd.find_objects_to_delete() # The 1's are purgeable (due to age of day set to 10) + self.assertEqual(len(objects_map.goals), 1) + self.assertEqual(len(objects_map.strategies), 1) self.assertEqual(len(objects_map.audit_templates), 1) self.assertEqual(len(objects_map.audits), 1) self.assertEqual(len(objects_map.action_plans), 1) @@ -186,9 +245,13 @@ class TestPurgeCommand(base.DbTestCase): def test_find_deleted_and_nonexpired_related_entries(self): with freezegun.freeze_time(self.fake_today): - # orphan audit + # orphan audit template + audit_template4 = obj_utils.create_test_audit_template( + self.context, goal_id=404, # Does not exist + name=self.generate_unique_name(prefix="Audit Template 4 "), + strategy_id=None, id=self._generate_id(), uuid=None) audit4 = obj_utils.create_test_audit( - self.context, audit_template_id=404, # Does not exist + self.context, audit_template_id=audit_template4.id, id=self._generate_id(), uuid=None) action_plan4 = obj_utils.create_test_action_plan( self.context, audit_id=audit4.id, @@ -197,8 +260,12 @@ class TestPurgeCommand(base.DbTestCase): self.context, action_plan_id=action_plan4.id, id=self._generate_id(), uuid=None) + audit_template5 = obj_utils.create_test_audit_template( + self.context, goal_id=self.goal1.id, + name=self.generate_unique_name(prefix="Audit Template 5 "), + strategy_id=None, id=self._generate_id(), uuid=None) audit5 = obj_utils.create_test_audit( - self.context, audit_template_id=self.audit_template1.id, + self.context, audit_template_id=audit_template5.id, id=self._generate_id(), uuid=None) action_plan5 = obj_utils.create_test_action_plan( self.context, audit_id=audit5.id, @@ -207,6 +274,8 @@ class TestPurgeCommand(base.DbTestCase): self.context, action_plan_id=action_plan5.id, id=self._generate_id(), uuid=None) + self.goal2.soft_delete() + self.strategy2.soft_delete() self.audit_template2.soft_delete() self.audit2.soft_delete() self.action_plan2.soft_delete() @@ -216,13 +285,16 @@ class TestPurgeCommand(base.DbTestCase): # All the 5's should be purged as well even though they are not # expired because their related audit template is itself expired + audit_template5.soft_delete() audit5.soft_delete() action_plan5.soft_delete() with freezegun.freeze_time(self.fake_today): objects_map = self.cmd.find_objects_to_delete() - self.assertEqual(len(objects_map.audit_templates), 1) + self.assertEqual(len(objects_map.goals), 1) + self.assertEqual(len(objects_map.strategies), 1) + self.assertEqual(len(objects_map.audit_templates), 3) self.assertEqual(len(objects_map.audits), 3) self.assertEqual(len(objects_map.action_plans), 3) self.assertEqual(len(objects_map.actions), 3) @@ -234,9 +306,11 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") - def test_purge_command(self, m_destroy_audit_template, - m_destroy_audit, m_destroy_action_plan, - m_destroy_action): + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") + def test_purge_command(self, m_destroy_goal, m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, + m_destroy_action_plan, m_destroy_action): with freezegun.freeze_time(self.fake_today): self.cmd.execute() @@ -253,13 +327,20 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") def test_purge_command_with_nonexpired_related_entries( - self, m_destroy_audit_template, m_destroy_audit, + self, m_destroy_goal, m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, m_destroy_action_plan, m_destroy_action): with freezegun.freeze_time(self.fake_today): - # orphan audit + # orphan audit template + audit_template4 = obj_utils.create_test_audit_template( + self.context, goal_id=404, # Does not exist + name=self.generate_unique_name(prefix="Audit Template 4 "), + strategy_id=None, id=self._generate_id(), uuid=None) audit4 = obj_utils.create_test_audit( - self.context, audit_template_id=404, # Does not exist + self.context, audit_template_id=audit_template4.id, id=self._generate_id(), uuid=None) action_plan4 = obj_utils.create_test_action_plan( self.context, audit_id=audit4.id, @@ -268,8 +349,12 @@ class TestPurgeCommand(base.DbTestCase): self.context, action_plan_id=action_plan4.id, id=self._generate_id(), uuid=None) + audit_template5 = obj_utils.create_test_audit_template( + self.context, goal_id=self.goal1.id, + name=self.generate_unique_name(prefix="Audit Template 5 "), + strategy_id=None, id=self._generate_id(), uuid=None) audit5 = obj_utils.create_test_audit( - self.context, audit_template_id=self.audit_template1.id, + self.context, audit_template_id=audit_template5.id, id=self._generate_id(), uuid=None) action_plan5 = obj_utils.create_test_action_plan( self.context, audit_id=audit5.id, @@ -278,6 +363,8 @@ class TestPurgeCommand(base.DbTestCase): self.context, action_plan_id=action_plan5.id, id=self._generate_id(), uuid=None) + self.goal2.soft_delete() + self.strategy2.soft_delete() self.audit_template2.soft_delete() self.audit2.soft_delete() self.action_plan2.soft_delete() @@ -287,13 +374,16 @@ class TestPurgeCommand(base.DbTestCase): # All the 5's should be purged as well even though they are not # expired because their related audit template is itself expired + audit_template5.soft_delete() audit5.soft_delete() action_plan5.soft_delete() with freezegun.freeze_time(self.fake_today): self.cmd.execute() - self.assertEqual(m_destroy_audit_template.call_count, 1) + self.assertEqual(m_destroy_goal.call_count, 1) + self.assertEqual(m_destroy_strategy.call_count, 1) + self.assertEqual(m_destroy_audit_template.call_count, 3) self.assertEqual(m_destroy_audit.call_count, 3) self.assertEqual(m_destroy_action_plan.call_count, 3) self.assertEqual(m_destroy_action.call_count, 3) @@ -312,8 +402,11 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") def test_purge_command_with_audit_template_ok( - self, m_destroy_audit_template, m_destroy_audit, + self, m_destroy_goal, m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, m_destroy_action_plan, m_destroy_action): self.cmd.orphans = False self.cmd.uuid = self.audit_template1.uuid @@ -321,6 +414,8 @@ class TestPurgeCommand(base.DbTestCase): with freezegun.freeze_time(self.fake_today): self.cmd.execute() + self.assertEqual(m_destroy_goal.call_count, 0) + self.assertEqual(m_destroy_strategy.call_count, 0) self.assertEqual(m_destroy_audit_template.call_count, 1) self.assertEqual(m_destroy_audit.call_count, 1) self.assertEqual(m_destroy_action_plan.call_count, 1) @@ -339,8 +434,11 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") def test_purge_command_with_audit_template_not_expired( - self, m_destroy_audit_template, m_destroy_audit, + self, m_destroy_goal, m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, m_destroy_action_plan, m_destroy_action): self.cmd.orphans = False self.cmd.uuid = self.audit_template2.uuid @@ -348,6 +446,8 @@ class TestPurgeCommand(base.DbTestCase): with freezegun.freeze_time(self.fake_today): self.cmd.execute() + self.assertEqual(m_destroy_goal.call_count, 0) + self.assertEqual(m_destroy_strategy.call_count, 0) self.assertEqual(m_destroy_audit_template.call_count, 0) self.assertEqual(m_destroy_audit.call_count, 0) self.assertEqual(m_destroy_action_plan.call_count, 0) @@ -357,8 +457,11 @@ class TestPurgeCommand(base.DbTestCase): @mock.patch.object(dbapi.Connection, "destroy_action_plan") @mock.patch.object(dbapi.Connection, "destroy_audit") @mock.patch.object(dbapi.Connection, "destroy_audit_template") + @mock.patch.object(dbapi.Connection, "destroy_strategy") + @mock.patch.object(dbapi.Connection, "destroy_goal") def test_purge_command_with_audit_template_not_soft_deleted( - self, m_destroy_audit_template, m_destroy_audit, + self, m_destroy_goal, m_destroy_strategy, + m_destroy_audit_template, m_destroy_audit, m_destroy_action_plan, m_destroy_action): self.cmd.orphans = False self.cmd.uuid = self.audit_template3.uuid @@ -366,6 +469,8 @@ class TestPurgeCommand(base.DbTestCase): with freezegun.freeze_time(self.fake_today): self.cmd.execute() + self.assertEqual(m_destroy_goal.call_count, 0) + self.assertEqual(m_destroy_strategy.call_count, 0) self.assertEqual(m_destroy_audit_template.call_count, 0) self.assertEqual(m_destroy_audit.call_count, 0) self.assertEqual(m_destroy_action_plan.call_count, 0)