Syncer now syncs stale audit templates

In this changeset, I introduce the syncing of audit templates.

Partially Implements: blueprint get-goal-from-strategy

Change-Id: Ie394c12fe51f73eff95465fd5140d82ebd212599
This commit is contained in:
Vincent Françoise 2016-04-18 14:20:09 +02:00
parent 2966b93777
commit 18e5c7d844
2 changed files with 364 additions and 59 deletions
watcher
decision_engine
tests/decision_engine

@ -14,22 +14,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from oslo_log import log
from watcher._i18n import _LI
from watcher._i18n import _LE, _LI
from watcher.common import context
from watcher.decision_engine.strategy.loading import default
from watcher import objects
LOG = log.getLogger(__name__)
GoalMapping = collections.namedtuple('GoalMapping', ['name', 'display_name'])
StrategyMapping = collections.namedtuple(
'StrategyMapping', ['name', 'goal_name', 'display_name'])
class Syncer(object):
"""Syncs all available goals and strategies with the Watcher DB"""
def __init__(self):
self.ctx = context.make_context()
self._discovered_map = None
self.discovered_map = None
self._available_goals = None
self._available_goals_map = None
@ -42,6 +48,8 @@ class Syncer(object):
# This strategy mapping maps stale strategy IDs to the synced goal
self.strategy_mapping = dict()
self.stale_audit_templates_map = {}
@property
def available_goals(self):
if self._available_goals is None:
@ -58,7 +66,8 @@ class Syncer(object):
def available_goals_map(self):
if self._available_goals_map is None:
self._available_goals_map = {
g: {"name": g.name, "display_name": g.display_name}
GoalMapping(
name=g.name, display_name=g.display_name): g
for g in self.available_goals
}
return self._available_goals_map
@ -68,41 +77,45 @@ class Syncer(object):
if self._available_strategies_map is None:
goals_map = {g.id: g.name for g in self.available_goals}
self._available_strategies_map = {
s: {"name": s.name, "goal_name": goals_map[s.goal_id],
"display_name": s.display_name}
StrategyMapping(
name=s.name, goal_name=goals_map[s.goal_id],
display_name=s.display_name): s
for s in self.available_strategies
}
return self._available_strategies_map
def sync(self):
discovered_map = self.discover()
goals_map = discovered_map["goals"]
strategies_map = discovered_map["strategies"]
self.discovered_map = self._discover()
goals_map = self.discovered_map["goals"]
strategies_map = self.discovered_map["strategies"]
for goal_name, goal_map in goals_map.items():
if goal_map in self.available_goals_map.values():
if goal_map in self.available_goals_map:
LOG.info(_LI("Goal %s already exists"), goal_name)
continue
self.goal_mapping.update(self._sync_goal(goal_map))
for strategy_name, strategy_map in strategies_map.items():
if strategy_map in self.available_strategies_map.values():
if (strategy_map in self.available_strategies_map and
strategy_map.goal_name not in
[g.name for g in self.goal_mapping.values()]):
LOG.info(_LI("Strategy %s already exists"), strategy_name)
continue
self.strategy_mapping.update(self._sync_strategy(strategy_map))
# TODO(v-francoise): Sync the audit templates
self._sync_audit_templates()
def _sync_goal(self, goal_map):
goal_name = goal_map['name']
goal_display_name = goal_map['display_name']
goal_name = goal_map.name
goal_display_name = goal_map.display_name
goal_mapping = dict()
# Goals that are matching by name with the given discovered goal name
matching_goals = [g for g in self.available_goals
if g.name == goal_name]
stale_goals = self.soft_delete_stale_goals(goal_map, matching_goals)
stale_goals = self._soft_delete_stale_goals(goal_map, matching_goals)
if stale_goals or not matching_goals:
goal = objects.Goal(self.ctx)
@ -120,14 +133,16 @@ class Syncer(object):
return goal_mapping
def _sync_strategy(self, strategy_map):
strategy_name = strategy_map['name']
strategy_display_name = strategy_map['display_name']
goal_name = strategy_map['goal_name']
strategy_name = strategy_map.name
strategy_display_name = strategy_map.display_name
goal_name = strategy_map.goal_name
strategy_mapping = dict()
# Strategies that are matching by name with the given
# discovered strategy name
matching_strategies = [s for s in self.available_strategies
if s.name == strategy_name]
stale_strategies = self.soft_delete_stale_strategies(
stale_strategies = self._soft_delete_stale_strategies(
strategy_map, matching_strategies)
if stale_strategies or not matching_strategies:
@ -146,7 +161,95 @@ class Syncer(object):
return strategy_mapping
def discover(self):
def _sync_audit_templates(self):
# First we find audit templates that are stale because their associated
# goal or strategy has been modified and we update them in-memory
self._find_stale_audit_templates_due_to_goal()
self._find_stale_audit_templates_due_to_strategy()
# Then we handle the case where an audit template became
# stale because its related goal does not exist anymore.
self._soft_delete_removed_goals()
# Then we handle the case where an audit template became
# stale because its related strategy does not exist anymore.
self._soft_delete_removed_strategies()
# Finally, we save into the DB the updated stale audit templates
for stale_audit_template in self.stale_audit_templates_map.values():
stale_audit_template.save()
LOG.info(_LI("Audit Template '%s' synced"),
stale_audit_template.name)
def _find_stale_audit_templates_due_to_goal(self):
for goal_id, synced_goal in self.goal_mapping.items():
filters = {"goal_id": goal_id}
stale_audit_templates = objects.AuditTemplate.list(
self.ctx, filters=filters)
# Update the goal ID for the stale audit templates (w/o saving)
for audit_template in stale_audit_templates:
if audit_template.id not in self.stale_audit_templates_map:
audit_template.goal_id = synced_goal.id
self.stale_audit_templates_map[audit_template.id] = (
audit_template)
else:
self.stale_audit_templates_map[
audit_template.id].goal_id = synced_goal.id
def _find_stale_audit_templates_due_to_strategy(self):
for strategy_id, synced_strategy in self.strategy_mapping.items():
filters = {"strategy_id": strategy_id}
stale_audit_templates = objects.AuditTemplate.list(
self.ctx, filters=filters)
# Update strategy IDs for all stale audit templates (w/o saving)
for audit_template in stale_audit_templates:
if audit_template.id not in self.stale_audit_templates_map:
audit_template.strategy_id = synced_strategy.id
self.stale_audit_templates_map[audit_template.id] = (
audit_template)
else:
self.stale_audit_templates_map[
audit_template.id].strategy_id = synced_strategy.id
def _soft_delete_removed_goals(self):
removed_goals = [
g for g in self.available_goals
if g.name not in self.discovered_map['goals']]
for removed_goal in removed_goals:
removed_goal.soft_delete()
filters = {"goal_id": removed_goal.id}
invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)
for at in invalid_ats:
LOG.warning(
_LE("Audit Template '%(audit_template)s' references a "
"goal that does not exist"),
audit_template=at.uuid)
def _soft_delete_removed_strategies(self):
removed_strategies = [
s for s in self.available_strategies
if s.name not in self.discovered_map['strategies']]
for removed_strategy in removed_strategies:
removed_strategy.soft_delete()
filters = {"strategy_id": removed_strategy.id}
invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)
for at in invalid_ats:
LOG.info(
_LI("Audit Template '%(audit_template)s' references a "
"strategy that does not exist"),
audit_template=at.uuid)
# In this case we can reset the strategy ID to None
# so the audit template can still achieve the same goal
# but with a different strategy
if at.id not in self.stale_audit_templates_map:
at.strategy_id = None
self.stale_audit_templates_map[at.id] = at
else:
self.stale_audit_templates_map[at.id].strategy_id = None
def _discover(self):
strategies_map = {}
goals_map = {}
discovered_map = {"goals": goals_map, "strategies": strategies_map}
@ -154,25 +257,25 @@ class Syncer(object):
implemented_strategies = strategy_loader.list_available()
for _, strategy_cls in implemented_strategies.items():
goals_map[strategy_cls.get_goal_name()] = {
"name": strategy_cls.get_goal_name(),
"display_name":
strategy_cls.get_translatable_goal_display_name()}
goals_map[strategy_cls.get_goal_name()] = GoalMapping(
name=strategy_cls.get_goal_name(),
display_name=strategy_cls.get_translatable_goal_display_name())
strategies_map[strategy_cls.get_name()] = {
"name": strategy_cls.get_name(),
"goal_name": strategy_cls.get_goal_name(),
"display_name": strategy_cls.get_translatable_display_name()}
strategies_map[strategy_cls.get_name()] = StrategyMapping(
name=strategy_cls.get_name(),
goal_name=strategy_cls.get_goal_name(),
display_name=strategy_cls.get_translatable_display_name())
return discovered_map
def soft_delete_stale_goals(self, goal_map, matching_goals):
goal_name = goal_map['name']
goal_display_name = goal_map['display_name']
def _soft_delete_stale_goals(self, goal_map, matching_goals):
goal_name = goal_map.name
goal_display_name = goal_map.display_name
stale_goals = []
for matching_goal in matching_goals:
if matching_goal.display_name == goal_display_name:
if (matching_goal.display_name == goal_display_name and
matching_goal.strategy_id not in self.strategy_mapping):
LOG.info(_LI("Goal %s unchanged"), goal_name)
else:
LOG.info(_LI("Goal %s modified"), goal_name)
@ -181,13 +284,14 @@ class Syncer(object):
return stale_goals
def soft_delete_stale_strategies(self, strategy_map, matching_strategies):
strategy_name = strategy_map['name']
strategy_display_name = strategy_map['display_name']
def _soft_delete_stale_strategies(self, strategy_map, matching_strategies):
strategy_name = strategy_map.name
strategy_display_name = strategy_map.display_name
stale_strategies = []
for matching_strategy in matching_strategies:
if matching_strategy.display_name == strategy_display_name:
if (matching_strategy.display_name == strategy_display_name and
matching_strategy.goal_id not in self.goal_mapping):
LOG.info(_LI("Strategy %s unchanged"), strategy_name)
else:
LOG.info(_LI("Strategy %s modified"), strategy_name)

@ -31,6 +31,7 @@ class TestSyncer(base.DbTestCase):
super(TestSyncer, self).setUp()
self.ctx = context.make_context()
# This mock simulates the strategies discovery done in discover()
self.m_available_strategies = mock.Mock(return_value={
fake_strategies.FakeDummy1Strategy1.get_name():
fake_strategies.FakeDummy1Strategy1,
@ -201,54 +202,254 @@ class TestSyncer(base.DbTestCase):
self.assertEqual(1, m_s_soft_delete.call_count)
def test_end2end_sync_goals_with_modified_goal_and_strategy(self):
goal = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Original")
goal.create()
strategy = objects.Strategy(
self.ctx, id=1, name="STRATEGY_1",
display_name="Original", goal_id=goal.id)
strategy.create()
# audit_template = objects.AuditTemplate(
# self.ctx, id=1, name="Synced AT", goal_id=goal.id,
# strategy_id=strategy.id)
# audit_template.create()
# ### Setup ### #
# before_audit_templates = objects.AuditTemplate.list(self.ctx)
# Here, we simulate goals and strategies already discovered in the past
# that were saved in DB
# Should stay unmodified after sync()
goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
# Should be modified by the sync()
goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="Original")
goal1.create()
goal2.create()
# Should stay unmodified after sync()
strategy1 = objects.Strategy(
self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(),
display_name="Strategy 1", goal_id=goal1.id)
# Should stay unmodified after sync()
strategy2 = objects.Strategy(
self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(),
display_name="Strategy 2", goal_id=goal2.id)
# Should be modified by the sync()
strategy3 = objects.Strategy(
self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(),
display_name="Original", goal_id=goal2.id)
# Should be modified by the sync()
strategy4 = objects.Strategy(
self.ctx, id=4, name="STRATEGY_4", uuid=utils.generate_uuid(),
display_name="Original", goal_id=goal2.id)
strategy1.create()
strategy2.create()
strategy3.create()
strategy4.create()
# Here we simulate audit_templates that were already created in the
# past and hence saved within the Watcher DB
# Should stay unmodified after sync()
audit_template1 = objects.AuditTemplate(
self.ctx, id=1, uuid=utils.generate_uuid(),
name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id)
# Should be modified by the sync() because its associated goal
# should be modified
audit_template2 = objects.AuditTemplate(
self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(),
goal_id=goal2.id, strategy_id=strategy2.id)
# Should be modified by the sync() because its associated strategy
# should be modified
audit_template3 = objects.AuditTemplate(
self.ctx, id=3, name="Synced AT3", uuid=utils.generate_uuid(),
goal_id=goal2.id, strategy_id=strategy3.id)
# Modified because of both because its associated goal and associated
# strategy should be modified
audit_template4 = objects.AuditTemplate(
self.ctx, id=4, name="Synced AT4", uuid=utils.generate_uuid(),
goal_id=goal2.id, strategy_id=strategy4.id)
audit_template1.create()
audit_template2.create()
audit_template3.create()
audit_template4.create()
before_audit_templates = objects.AuditTemplate.list(self.ctx)
before_goals = objects.Goal.list(self.ctx)
before_strategies = objects.Strategy.list(self.ctx)
# ### Action under test ### #
try:
self.syncer.sync()
except Exception as exc:
self.fail(exc)
# after_audit_templates = objects.AuditTemplate.list(self.ctx)
# ### Assertions ### #
after_audit_templates = objects.AuditTemplate.list(self.ctx)
after_goals = objects.Goal.list(self.ctx)
after_strategies = objects.Strategy.list(self.ctx)
self.assertEqual(1, len(before_goals))
self.assertEqual(1, len(before_strategies))
# self.assertEqual(1, len(before_audit_templates))
self.assertEqual(2, len(before_goals))
self.assertEqual(4, len(before_strategies))
self.assertEqual(4, len(before_audit_templates))
self.assertEqual(2, len(after_goals))
self.assertEqual(4, len(after_strategies))
# self.assertEqual(1, len(after_audit_templates))
self.assertEqual(4, len(after_audit_templates))
self.assertEqual(
{"DUMMY_1", "DUMMY_2"},
set([g.name for g in after_goals]))
self.assertEqual(
{"STRATEGY_1", "STRATEGY_2", "STRATEGY_3", "STRATEGY_4"},
set([s.name for s in after_strategies]))
created_goals = {
ag.name: ag for ag in after_goals
if ag.uuid not in [bg.uuid for bg in before_goals]
}
created_strategies = {
a_s.name: a_s for a_s in after_strategies
if a_s.uuid not in [b_s.uuid for b_s in before_strategies]
}
self.assertEqual(1, len(created_goals))
self.assertEqual(3, len(created_strategies))
modified_audit_templates = {
a_at.id for a_at in after_audit_templates
if a_at.goal_id not in (
# initial goal IDs
b_at.goal_id for b_at in before_audit_templates) or
a_at.strategy_id not in (
# initial strategy IDs
b_at.strategy_id for b_at in before_audit_templates
if b_at.strategy_id is not None)
}
unmodified_audit_templates = {
a_at.id for a_at in after_audit_templates
if a_at.goal_id in (
# initial goal IDs
b_at.goal_id for b_at in before_audit_templates) and
a_at.strategy_id in (
# initial strategy IDs
b_at.strategy_id for b_at in before_audit_templates
if b_at.strategy_id is not None)
}
self.assertEqual(2, strategy2.goal_id)
self.assertIn(strategy2.name, created_strategies)
self.assertTrue(strategy2.id != created_strategies[strategy2.name].id)
self.assertEqual(set([audit_template2.id,
audit_template3.id,
audit_template4.id]),
modified_audit_templates)
self.assertEqual(set([audit_template1.id]),
unmodified_audit_templates)
def test_end2end_sync_goals_with_removed_goal_and_strategy(self):
# ### Setup ### #
# We Simulate the fact that we removed 2 strategies
# as well as the DUMMY_2 goal
self.m_available_strategies.return_value = {
fake_strategies.FakeDummy1Strategy1.get_name():
fake_strategies.FakeDummy1Strategy1
}
# Should stay unmodified after sync()
goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
# To be removed by the sync()
goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="Dummy 2")
goal1.create()
goal2.create()
# Should stay unmodified after sync()
strategy1 = objects.Strategy(
self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(),
display_name="Strategy 1", goal_id=goal1.id)
# To be removed by the sync()
strategy2 = objects.Strategy(
self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(),
display_name="Strategy 2", goal_id=goal1.id)
# To be removed by the sync()
strategy3 = objects.Strategy(
self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(),
display_name="Original", goal_id=goal2.id)
strategy1.create()
strategy2.create()
strategy3.create()
# Here we simulate audit_templates that were already created in the
# past and hence saved within the Watcher DB
# The strategy of this audit template will be dereferenced
# as it does not exist anymore
audit_template1 = objects.AuditTemplate(
self.ctx, id=1, uuid=utils.generate_uuid(),
name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id)
# Stale even after syncing because the goal has been soft deleted
audit_template2 = objects.AuditTemplate(
self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(),
goal_id=goal2.id, strategy_id=strategy2.id)
audit_template1.create()
audit_template2.create()
before_audit_templates = objects.AuditTemplate.list(self.ctx)
before_goals = objects.Goal.list(self.ctx)
before_strategies = objects.Strategy.list(self.ctx)
# ### Action under test ### #
try:
self.syncer.sync()
except Exception as exc:
self.fail(exc)
# ### Assertions ### #
after_audit_templates = objects.AuditTemplate.list(self.ctx)
after_goals = objects.Goal.list(self.ctx)
after_strategies = objects.Strategy.list(self.ctx)
self.assertEqual(2, len(before_goals))
self.assertEqual(3, len(before_strategies))
self.assertEqual(2, len(before_audit_templates))
self.assertEqual(1, len(after_goals))
self.assertEqual(1, len(after_strategies))
self.assertEqual(2, len(after_audit_templates))
self.assertEqual(
{"DUMMY_1"},
set([g.name for g in after_goals]))
self.assertEqual(
{"STRATEGY_1"},
set([s.name for s in after_strategies]))
created_goals = [ag for ag in after_goals
if ag.uuid not in [bg.uuid for bg in before_goals]]
created_strategies = [
a_s for a_s in after_strategies
if a_s.uuid not in [b_s.uuid for b_s in before_strategies]]
self.assertEqual(2, len(created_goals))
self.assertEqual(4, len(created_strategies))
self.assertEqual(0, len(created_goals))
self.assertEqual(0, len(created_strategies))
# synced_audit_template = after_audit_templates[0]
# self.assertTrue(
# audit_template.goal_id != synced_audit_template.goal_id)
# self.assertIn(synced_audit_template.goal_id,
# (g.id for g in after_goals))
modified_audit_templates = {
a_at.id for a_at in after_audit_templates
if a_at.goal_id not in (
# initial goal IDs
b_at.goal_id for b_at in before_audit_templates) or
a_at.strategy_id not in (
# initial strategy IDs
b_at.strategy_id for b_at in before_audit_templates
if b_at.strategy_id is not None)
}
unmodified_audit_templates = {
a_at.id for a_at in after_audit_templates
if a_at.goal_id in (
# initial goal IDs
b_at.goal_id for b_at in before_audit_templates) and
a_at.strategy_id in (
# initial strategy IDs
b_at.strategy_id for b_at in before_audit_templates
if b_at.strategy_id is not None)
}
self.assertEqual(set([audit_template2.id]),
modified_audit_templates)
self.assertEqual(set([audit_template1.id]),
unmodified_audit_templates)