From 48cc6b271849687bb088b9f71e50a7bebf12ca3b Mon Sep 17 00:00:00 2001 From: Alexander Chadin Date: Wed, 24 Aug 2016 18:28:19 +0300 Subject: [PATCH] Add Audit Scope Handler This patch set adds audit scope mechanism. It also removes host_aggregate field. Change-Id: Ia98ed180a93fc8c19599735e2b41471d322bae9a Partially-Implements: blueprint define-the-audit-scope --- watcher/api/controllers/v1/audit.py | 31 ++- watcher/api/controllers/v1/audit_template.py | 25 +- watcher/common/exception.py | 5 + watcher/common/nova_helper.py | 11 +- watcher/common/utils.py | 2 + watcher/db/api.py | 1 - watcher/db/sqlalchemy/api.py | 3 +- watcher/db/sqlalchemy/models.py | 4 +- watcher/decision_engine/audit/base.py | 11 +- .../decision_engine/model/collector/base.py | 5 +- watcher/decision_engine/scope/__init__.py | 0 watcher/decision_engine/scope/base.py | 38 +++ watcher/decision_engine/scope/default.py | 219 ++++++++++++++++++ .../strategy/context/default.py | 2 + .../strategy/strategies/base.py | 23 +- watcher/objects/audit.py | 2 +- watcher/objects/audit_template.py | 2 +- watcher/tests/api/v1/test_audit_templates.py | 3 +- watcher/tests/api/v1/test_audits.py | 17 +- watcher/tests/db/test_audit_template.py | 10 - watcher/tests/db/utils.py | 4 +- .../audit/test_audit_handlers.py | 2 +- .../tests/decision_engine/scope/__init__.py | 0 .../decision_engine/scope/fake_scopes.py | 35 +++ .../decision_engine/scope/test_default.py | 210 +++++++++++++++++ .../strategies/test_basic_consolidation.py | 16 +- .../strategies/test_dummy_strategy.py | 9 + .../strategies/test_outlet_temp_control.py | 9 + .../strategies/test_uniform_airflow.py | 9 + .../test_vm_workload_consolidation.py | 9 + .../strategies/test_workload_balance.py | 9 + .../strategies/test_workload_stabilization.py | 8 + .../services/infra_optim/v1/json/client.py | 3 - .../tests/api/admin/base.py | 9 +- .../tests/api/admin/test_audit_template.py | 19 -- watcher_tempest_plugin/tests/scenario/base.py | 9 +- 36 files changed, 673 insertions(+), 101 deletions(-) create mode 100644 watcher/decision_engine/scope/__init__.py create mode 100644 watcher/decision_engine/scope/base.py create mode 100644 watcher/decision_engine/scope/default.py create mode 100644 watcher/tests/decision_engine/scope/__init__.py create mode 100644 watcher/tests/decision_engine/scope/fake_scopes.py create mode 100644 watcher/tests/decision_engine/scope/test_default.py diff --git a/watcher/api/controllers/v1/audit.py b/watcher/api/controllers/v1/audit.py index 743bdc8c9..3a9de3b70 100644 --- a/watcher/api/controllers/v1/audit.py +++ b/watcher/api/controllers/v1/audit.py @@ -54,6 +54,8 @@ class AuditPostType(wtypes.Base): audit_template_uuid = wtypes.wsattr(types.uuid, mandatory=False) + scope = wtypes.wsattr(types.jsontype, readonly=True) + goal = wtypes.wsattr(wtypes.text, mandatory=False) strategy = wtypes.wsattr(wtypes.text, mandatory=False) @@ -69,9 +71,6 @@ class AuditPostType(wtypes.Base): default={}) interval = wsme.wsattr(int, mandatory=False) - host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1), - mandatory=False) - def as_audit(self, context): audit_type_values = [val.value for val in objects.audit.AuditType] if self.audit_type not in audit_type_values: @@ -100,7 +99,7 @@ class AuditPostType(wtypes.Base): at2a = { 'goal': 'goal_id', 'strategy': 'strategy_id', - 'host_aggregate': 'host_aggregate' + 'scope': 'scope', } to_string_fields = set(['goal', 'strategy']) for k in at2a: @@ -117,9 +116,9 @@ class AuditPostType(wtypes.Base): deadline=self.deadline, parameters=self.parameters, goal_id=self.goal, - host_aggregate=self.host_aggregate, strategy_id=self.strategy, - interval=self.interval) + interval=self.interval, + scope=self.scope,) class AuditPatchType(types.JsonPatchType): @@ -261,8 +260,8 @@ class Audit(base.APIBase): interval = wsme.wsattr(int, mandatory=False) """Launch audit periodically (in seconds)""" - host_aggregate = wtypes.IntegerType(minimum=1) - """ID of the Nova host aggregate targeted by the audit template""" + scope = wsme.wsattr(types.jsontype, mandatory=False) + """Audit Scope""" def __init__(self, **kwargs): self.fields = [] @@ -294,8 +293,8 @@ class Audit(base.APIBase): if not expand: audit.unset_fields_except(['uuid', 'audit_type', 'deadline', 'state', 'goal_uuid', 'interval', - 'strategy_uuid', 'host_aggregate', - 'goal_name', 'strategy_name']) + 'strategy_uuid', 'goal_name', + 'strategy_name']) audit.links = [link.Link.make_link('self', url, 'audits', audit.uuid), @@ -320,11 +319,11 @@ class Audit(base.APIBase): created_at=datetime.datetime.utcnow(), deleted_at=None, updated_at=datetime.datetime.utcnow(), - interval=7200) + interval=7200, + scope=[]) sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae' sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff' - sample.host_aggregate = 1 return cls._convert_with_links(sample, 'http://localhost:9322', expand) @@ -381,7 +380,7 @@ class AuditsController(rest.RestController): def _get_audits_collection(self, marker, limit, sort_key, sort_dir, expand=False, resource_url=None, goal=None, - strategy=None, host_aggregate=None): + strategy=None): limit = api_utils.validate_limit(limit) api_utils.validate_sort_dir(sort_dir) marker_obj = None @@ -426,7 +425,7 @@ class AuditsController(rest.RestController): wtypes.text, wtypes.text, wtypes.text, int) def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc', goal=None, - strategy=None, host_aggregate=None): + strategy=None): """Retrieve a list of audits. :param marker: pagination marker for large data sets. @@ -436,7 +435,6 @@ class AuditsController(rest.RestController): id. :param goal: goal UUID or name to filter by :param strategy: strategy UUID or name to filter by - :param host_aggregate: Optional host_aggregate """ context = pecan.request.context @@ -445,8 +443,7 @@ class AuditsController(rest.RestController): return self._get_audits_collection(marker, limit, sort_key, sort_dir, goal=goal, - strategy=strategy, - host_aggregate=host_aggregate) + strategy=strategy) @wsme_pecan.wsexpose(AuditCollection, wtypes.text, types.uuid, int, wtypes.text, wtypes.text) diff --git a/watcher/api/controllers/v1/audit_template.py b/watcher/api/controllers/v1/audit_template.py index d004e6138..bd417ded9 100644 --- a/watcher/api/controllers/v1/audit_template.py +++ b/watcher/api/controllers/v1/audit_template.py @@ -66,6 +66,7 @@ from watcher.common import context as context_utils from watcher.common import exception from watcher.common import policy from watcher.common import utils as common_utils +from watcher.decision_engine.scope import default from watcher import objects @@ -81,10 +82,6 @@ class AuditTemplatePostType(wtypes.Base): deadline = wsme.wsattr(datetime.datetime, mandatory=False) """deadline of the audit template""" - host_aggregate = wsme.wsattr(wtypes.IntegerType(minimum=1), - mandatory=False) - """ID of the Nova host aggregate targeted by the audit template""" - extra = wtypes.wsattr({wtypes.text: types.jsontype}, mandatory=False) """The metadata of the audit template""" @@ -97,18 +94,21 @@ class AuditTemplatePostType(wtypes.Base): version = wtypes.text """Internal version of the audit template""" + scope = wtypes.wsattr(types.jsontype, mandatory=False, default=[]) + """Audit Scope""" + def as_audit_template(self): return AuditTemplate( name=self.name, description=self.description, deadline=self.deadline, - host_aggregate=self.host_aggregate, extra=self.extra, goal_id=self.goal, # Dirty trick ... goal=self.goal, strategy_id=self.strategy, # Dirty trick ... strategy_uuid=self.strategy, version=self.version, + scope=self.scope, ) @staticmethod @@ -123,6 +123,9 @@ class AuditTemplatePostType(wtypes.Base): else: raise exception.InvalidGoal(goal=audit_template.goal) + common_utils.Draft4Validator( + default.DefaultScope.DEFAULT_SCHEMA).validate(audit_template.scope) + if audit_template.strategy: available_strategies = objects.Strategy.list( AuditTemplatePostType._ctx) @@ -311,9 +314,6 @@ class AuditTemplate(base.APIBase): deadline = datetime.datetime """deadline of the audit template""" - host_aggregate = wtypes.IntegerType(minimum=1) - """ID of the Nova host aggregate targeted by the audit template""" - extra = {wtypes.text: types.jsontype} """The metadata of the audit template""" @@ -342,6 +342,9 @@ class AuditTemplate(base.APIBase): links = wsme.wsattr([link.Link], readonly=True) """A list containing a self link and associated audit template links""" + scope = wsme.wsattr(types.jsontype, mandatory=False) + """Audit Scope""" + def __init__(self, **kwargs): super(AuditTemplate, self).__init__() self.fields = [] @@ -374,7 +377,7 @@ class AuditTemplate(base.APIBase): def _convert_with_links(audit_template, url, expand=True): if not expand: audit_template.unset_fields_except( - ['uuid', 'name', 'host_aggregate', 'goal_uuid', 'goal_name', + ['uuid', 'name', 'goal_uuid', 'goal_name', 'strategy_uuid', 'strategy_name']) # The numeric ID should not be exposed to @@ -402,13 +405,13 @@ class AuditTemplate(base.APIBase): sample = cls(uuid='27e3153e-d5bf-4b7e-b517-fb518e17f34c', name='My Audit Template', description='Description of my audit template', - host_aggregate=5, goal_uuid='83e44733-b640-40e2-8d8a-7dd3be7134e6', strategy_uuid='367d826e-b6a4-4b70-bc44-c3f6fe1c9986', extra={'automatic': True}, created_at=datetime.datetime.utcnow(), deleted_at=None, - updated_at=datetime.datetime.utcnow()) + updated_at=datetime.datetime.utcnow(), + scope=[],) return cls._convert_with_links(sample, 'http://localhost:9322', expand) diff --git a/watcher/common/exception.py b/watcher/common/exception.py index 8890684be..01ee74ba9 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -370,6 +370,11 @@ class ServiceNotFound(ResourceNotFound): msg_fmt = _("The service %(service)s cannot be found.") +class WildcardCharacterIsUsed(WatcherException): + msg_fmt = _("You shouldn't use any other IDs of %(resource)s if you use " + "wildcard character.") + + # Model class InstanceNotFound(WatcherException): diff --git a/watcher/common/nova_helper.py b/watcher/common/nova_helper.py index f7f296fc5..c8aee3066 100644 --- a/watcher/common/nova_helper.py +++ b/watcher/common/nova_helper.py @@ -63,6 +63,15 @@ class NovaHelper(object): LOG.exception(exc) raise exception.ComputeNodeNotFound(name=node_hostname) + def get_aggregate_list(self): + return self.nova.aggregates.list() + + def get_aggregate_detail(self, aggregate_id): + return self.nova.aggregates.get(aggregate_id) + + def get_availability_zone_list(self): + return self.nova.availability_zones.list(detailed=True) + def find_instance(self, instance_id): search_opts = {'all_tenants': True} instances = self.nova.servers.list(detailed=True, @@ -663,7 +672,7 @@ class NovaHelper(object): cache[fid] = flavor attr_defaults = [('name', 'unknown-id-%s' % fid), ('vcpus', 0), ('ram', 0), ('disk', 0), - ('ephemeral', 0)] + ('ephemeral', 0), ('extra_specs', {})] for attr, default in attr_defaults: if not flavor: instance.flavor[attr] = default diff --git a/watcher/common/utils.py b/watcher/common/utils.py index c2f3b1e81..0ba98db76 100644 --- a/watcher/common/utils.py +++ b/watcher/common/utils.py @@ -152,3 +152,5 @@ def extend_with_strict_schema(validator_class): StrictDefaultValidatingDraft4Validator = extend_with_default( extend_with_strict_schema(validators.Draft4Validator)) + +Draft4Validator = validators.Draft4Validator diff --git a/watcher/db/api.py b/watcher/db/api.py index 2d7f843c5..f257ec485 100644 --- a/watcher/db/api.py +++ b/watcher/db/api.py @@ -250,7 +250,6 @@ class BaseConnection(object): 'uuid': utils.generate_uuid(), 'name': 'example', 'description': 'free text description' - 'host_aggregate': 'nova aggregate name or id' 'goal': 'DUMMY' 'extra': {'automatic': True} } diff --git a/watcher/db/sqlalchemy/api.py b/watcher/db/sqlalchemy/api.py index 26caef3c6..8be7bb326 100644 --- a/watcher/db/sqlalchemy/api.py +++ b/watcher/db/sqlalchemy/api.py @@ -309,8 +309,7 @@ class Connection(api.BaseConnection): if filters is None: filters = {} - plain_fields = ['uuid', 'name', 'host_aggregate', - 'goal_id', 'strategy_id'] + plain_fields = ['uuid', 'name', 'goal_id', 'strategy_id'] join_fieldmap = JoinMap( goal_uuid=NaturalJoinFilter( join_fieldname="uuid", join_model=models.Goal), diff --git a/watcher/db/sqlalchemy/models.py b/watcher/db/sqlalchemy/models.py index 53dab4ba4..90157973e 100644 --- a/watcher/db/sqlalchemy/models.py +++ b/watcher/db/sqlalchemy/models.py @@ -157,11 +157,11 @@ class AuditTemplate(Base): uuid = Column(String(36)) name = Column(String(63), nullable=True) description = Column(String(255), nullable=True) - host_aggregate = Column(Integer, nullable=True) goal_id = Column(Integer, ForeignKey('goals.id'), nullable=False) strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=True) extra = Column(JSONEncodedDict) version = Column(String(15), nullable=True) + scope = Column(JSONEncodedList) class Audit(Base): @@ -179,9 +179,9 @@ class Audit(Base): deadline = Column(DateTime, nullable=True) parameters = Column(JSONEncodedDict, nullable=True) interval = Column(Integer, nullable=True) - host_aggregate = Column(Integer, nullable=True) goal_id = Column(Integer, ForeignKey('goals.id'), nullable=False) strategy_id = Column(Integer, ForeignKey('strategies.id'), nullable=True) + scope = Column(JSONEncodedList, nullable=True) class Action(Base): diff --git a/watcher/decision_engine/audit/base.py b/watcher/decision_engine/audit/base.py index b857c7678..cb6a109d5 100644 --- a/watcher/decision_engine/audit/base.py +++ b/watcher/decision_engine/audit/base.py @@ -80,7 +80,7 @@ class AuditHandler(BaseAuditHandler): 'audit_status': status} self.messaging.publish_status_event(event.type.name, payload) - def update_audit_state(self, request_context, audit, state): + def update_audit_state(self, audit, state): LOG.debug("Update audit state: %s", state) audit.state = state audit.save() @@ -89,15 +89,13 @@ class AuditHandler(BaseAuditHandler): def pre_execute(self, audit, request_context): LOG.debug("Trigger audit %s", audit.uuid) # change state of the audit to ONGOING - self.update_audit_state(request_context, audit, - audit_objects.State.ONGOING) + self.update_audit_state(audit, audit_objects.State.ONGOING) def post_execute(self, audit, solution, request_context): self.planner.schedule(request_context, audit.id, solution) # change state of the audit to SUCCEEDED - self.update_audit_state(request_context, audit, - audit_objects.State.SUCCEEDED) + self.update_audit_state(audit, audit_objects.State.SUCCEEDED) def execute(self, audit, request_context): try: @@ -106,5 +104,4 @@ class AuditHandler(BaseAuditHandler): self.post_execute(audit, solution, request_context) except Exception as e: LOG.exception(e) - self.update_audit_state(request_context, audit, - audit_objects.State.FAILED) + self.update_audit_state(audit, audit_objects.State.FAILED) diff --git a/watcher/decision_engine/model/collector/base.py b/watcher/decision_engine/model/collector/base.py index 18f62e5ff..28e9f2141 100644 --- a/watcher/decision_engine/model/collector/base.py +++ b/watcher/decision_engine/model/collector/base.py @@ -31,9 +31,8 @@ to know the current relationships between the different :ref:`resources during an :ref:`Audit ` and enables the :ref:`Strategy ` to request information such as: -- What compute nodes are in a given :ref:`Availability Zone - ` or a given :ref:`Host Aggregate - `? +- What compute nodes are in a given :ref:`Audit Scope + `? - What :ref:`Instances ` are hosted on a given compute node? - What is the current load of a compute node? diff --git a/watcher/decision_engine/scope/__init__.py b/watcher/decision_engine/scope/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/decision_engine/scope/base.py b/watcher/decision_engine/scope/base.py new file mode 100644 index 000000000..69c14b8fc --- /dev/null +++ b/watcher/decision_engine/scope/base.py @@ -0,0 +1,38 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 Servionica +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc +import six + +from watcher.common import context + + +@six.add_metaclass(abc.ABCMeta) +class BaseScope(object): + """A base class for Scope mechanism + + Child of this class is called when audit launches strategy. This strategy + requires Cluster Data Model which can be segregated to achieve audit scope. + """ + + def __init__(self, scope): + self.ctx = context.make_context() + self.scope = scope + + @abc.abstractmethod + def get_scoped_model(self, cluster_model): + """Leave only nodes and instances proposed in the audit scope""" diff --git a/watcher/decision_engine/scope/default.py b/watcher/decision_engine/scope/default.py new file mode 100644 index 000000000..f9c64fd56 --- /dev/null +++ b/watcher/decision_engine/scope/default.py @@ -0,0 +1,219 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 Servionica +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import copy + +from oslo_log import log + +from watcher._i18n import _LW +from watcher.common import exception +from watcher.common import nova_helper +from watcher.decision_engine.scope import base + + +LOG = log.getLogger(__name__) + + +class DefaultScope(base.BaseScope): + """Default Audit Scope Handler""" + + DEFAULT_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "host_aggregates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "anyOf": [ + {"type": ["string", "number"]} + ] + }, + } + }, + "availability_zones": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "additionalProperties": False + } + }, + "exclude": { + "type": "array", + "items": { + "type": "object", + "properties": { + "instances": { + "type": "array", + "items": { + "type": "object", + "properties": { + "uuid": { + "type": "string" + } + } + } + }, + "compute_nodes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + } + }, + "additionalProperties": False + } + } + }, + "additionalProperties": False + } + } + + def __init__(self, scope, osc=None): + super(DefaultScope, self).__init__(scope) + self._osc = osc + self.wrapper = nova_helper.NovaHelper(osc=self._osc) + + def _remove_instance(self, cluster_model, instance_uuid, node_name): + node = cluster_model.get_node_by_uuid(node_name) + instance = cluster_model.get_instance_by_uuid(instance_uuid) + cluster_model.delete_instance(instance, node) + + def _check_wildcard(self, aggregate_list): + if '*' in aggregate_list: + if len(aggregate_list) == 1: + return True + else: + raise exception.WildcardCharacterIsUsed( + resource="host aggregates") + return False + + def _collect_aggregates(self, host_aggregates, allowed_nodes): + aggregate_list = self.wrapper.get_aggregate_list() + aggregate_ids = [aggregate['id'] for aggregate + in host_aggregates if 'id' in aggregate] + aggregate_names = [aggregate['name'] for aggregate + in host_aggregates if 'name' in aggregate] + include_all_nodes = any(self._check_wildcard(field) + for field in (aggregate_ids, aggregate_names)) + + for aggregate in aggregate_list: + detailed_aggregate = self.wrapper.get_aggregate_detail( + aggregate.id) + if (detailed_aggregate.id in aggregate_ids or + detailed_aggregate.name in aggregate_names or + include_all_nodes): + allowed_nodes.extend(detailed_aggregate.hosts) + + def _collect_zones(self, availability_zones, allowed_nodes): + zone_list = self.wrapper.get_availability_zone_list() + zone_names = [zone['name'] for zone + in availability_zones] + include_all_nodes = False + if '*' in zone_names: + if len(zone_names) == 1: + include_all_nodes = True + else: + raise exception.WildcardCharacterIsUsed( + resource="availability zones") + for zone in zone_list: + if zone.zoneName in zone_names or include_all_nodes: + allowed_nodes.extend(zone.hosts.keys()) + + def _exclude_resources(self, resources, **kwargs): + instances_to_exclude = kwargs.get('instances') + nodes_to_exclude = kwargs.get('nodes') + for resource in resources: + if 'instances' in resource: + instances_to_exclude.extend( + [instance['uuid'] for instance + in resource['instances']]) + elif 'compute_nodes' in resource: + nodes_to_exclude.extend( + [host['name'] for host + in resource['compute_nodes']]) + + def _remove_node_from_model(self, nodes_to_remove, cluster_model): + for node_name in nodes_to_remove: + instances = copy.copy( + cluster_model.get_mapping().get_node_instances_by_uuid( + node_name)) + for instance_uuid in instances: + self._remove_instance(cluster_model, instance_uuid, node_name) + node = cluster_model.get_node_by_uuid(node_name) + cluster_model.remove_node(node) + + def _remove_instances_from_model(self, instances_to_remove, cluster_model): + for instance_uuid in instances_to_remove: + try: + node_name = (cluster_model.get_mapping() + .get_node_by_instance_uuid(instance_uuid).uuid) + except KeyError: + LOG.warning(_LW("The following instance %s cannot be found. " + "It might be deleted from CDM along with node" + " instance was hosted on."), + instance_uuid) + continue + self._remove_instance(cluster_model, instance_uuid, node_name) + + def get_scoped_model(self, cluster_model): + """Leave only nodes and instances proposed in the audit scope""" + + if not cluster_model: + return None + + allowed_nodes = [] + nodes_to_exclude = [] + instances_to_exclude = [] + model_hosts = list(cluster_model.get_all_compute_nodes().keys()) + + if not self.scope: + return cluster_model + + for rule in self.scope: + if 'host_aggregates' in rule: + self._collect_aggregates(rule['host_aggregates'], + allowed_nodes) + elif 'availability_zones' in rule: + self._collect_zones(rule['availability_zones'], + allowed_nodes) + elif 'exclude' in rule: + self._exclude_resources( + rule['exclude'], instances=instances_to_exclude, + nodes=nodes_to_exclude) + + instances_to_remove = set(instances_to_exclude) + nodes_to_remove = set(model_hosts) - set(allowed_nodes) + nodes_to_remove.update(nodes_to_exclude) + + self._remove_node_from_model(nodes_to_remove, cluster_model) + self._remove_instances_from_model(instances_to_remove, cluster_model) + + return cluster_model diff --git a/watcher/decision_engine/strategy/context/default.py b/watcher/decision_engine/strategy/context/default.py index cfd433b88..e20797372 100644 --- a/watcher/decision_engine/strategy/context/default.py +++ b/watcher/decision_engine/strategy/context/default.py @@ -53,6 +53,8 @@ class DefaultStrategyContext(base.BaseStrategyContext): selected_strategy = strategy_selector.select() + selected_strategy.audit_scope = audit.scope + schema = selected_strategy.get_schema() if not audit.parameters and schema: # Default value feedback if no predefined strategy diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index de41a837c..b1b552ac2 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -40,11 +40,13 @@ import abc import six from watcher.common import clients +from watcher.common import context from watcher.common import exception from watcher.common.loader import loadable from watcher.common import utils from watcher.decision_engine.loading import default as loading from watcher.decision_engine.model.collector import manager +from watcher.decision_engine.scope import default as default_scope from watcher.decision_engine.solution import default from watcher.decision_engine.strategy.common import level @@ -66,6 +68,7 @@ class BaseStrategy(loadable.Loadable): :type osc: :py:class:`~.OpenStackClients` instance """ super(BaseStrategy, self).__init__(config) + self.ctx = context.make_context() self._name = self.get_name() self._display_name = self.get_display_name() self._goal = self.get_goal() @@ -78,6 +81,8 @@ class BaseStrategy(loadable.Loadable): self._collector_manager = None self._compute_model = None self._input_parameters = utils.Struct() + self._audit_scope = None + self._audit_scope_handler = None @classmethod @abc.abstractmethod @@ -174,7 +179,8 @@ class BaseStrategy(loadable.Loadable): if self._compute_model is None: collector = self.collector_manager.get_cluster_model_collector( 'compute', osc=self.osc) - self._compute_model = collector.get_latest_cluster_data_model() + self._compute_model = self.audit_scope_handler.get_scoped_model( + collector.get_latest_cluster_data_model()) if not self._compute_model: raise exception.ClusterStateNotDefined() @@ -212,6 +218,21 @@ class BaseStrategy(loadable.Loadable): def solution(self, s): self._solution = s + @property + def audit_scope(self): + return self._audit_scope + + @audit_scope.setter + def audit_scope(self, s): + self._audit_scope = s + + @property + def audit_scope_handler(self): + if not self._audit_scope_handler: + self._audit_scope_handler = default_scope.DefaultScope( + self.audit_scope) + return self._audit_scope_handler + @property def name(self): return self._name diff --git a/watcher/objects/audit.py b/watcher/objects/audit.py index 1c31b84f9..f6ba5fbaf 100644 --- a/watcher/objects/audit.py +++ b/watcher/objects/audit.py @@ -88,7 +88,7 @@ class Audit(base.WatcherObject): 'interval': obj_utils.int_or_none, 'goal_id': obj_utils.int_or_none, 'strategy_id': obj_utils.int_or_none, - 'host_aggregate': obj_utils.int_or_none, + 'scope': obj_utils.list_or_none, } @staticmethod diff --git a/watcher/objects/audit_template.py b/watcher/objects/audit_template.py index 2a3a4f255..1ff45c84e 100644 --- a/watcher/objects/audit_template.py +++ b/watcher/objects/audit_template.py @@ -67,9 +67,9 @@ class AuditTemplate(base.WatcherObject): 'description': obj_utils.str_or_none, 'goal_id': obj_utils.int_or_none, 'strategy_id': obj_utils.int_or_none, - 'host_aggregate': obj_utils.int_or_none, 'extra': obj_utils.dict_or_none, 'version': obj_utils.str_or_none, + 'scope': obj_utils.list_or_none, } @staticmethod diff --git a/watcher/tests/api/v1/test_audit_templates.py b/watcher/tests/api/v1/test_audit_templates.py index c526a2001..752f6722e 100644 --- a/watcher/tests/api/v1/test_audit_templates.py +++ b/watcher/tests/api/v1/test_audit_templates.py @@ -81,8 +81,7 @@ class TestListAuditTemplate(FunctionalTestWithSetup): def _assert_audit_template_fields(self, audit_template): audit_template_fields = ['name', 'goal_uuid', 'goal_name', - 'strategy_uuid', 'strategy_name', - 'host_aggregate'] + 'strategy_uuid', 'strategy_name'] for field in audit_template_fields: self.assertIn(field, audit_template) diff --git a/watcher/tests/api/v1/test_audits.py b/watcher/tests/api/v1/test_audits.py index 23b315e40..e5c79878e 100644 --- a/watcher/tests/api/v1/test_audits.py +++ b/watcher/tests/api/v1/test_audits.py @@ -88,7 +88,7 @@ class TestListAudit(api_base.FunctionalTest): def _assert_audit_fields(self, audit): audit_fields = ['audit_type', 'deadline', 'state', 'goal_uuid', - 'strategy_uuid', 'host_aggregate'] + 'strategy_uuid'] for field in audit_fields: self.assertIn(field, audit) @@ -369,6 +369,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict) self.assertEqual('application/json', response.content_type) @@ -410,6 +411,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] # Make the audit template UUID some garbage value audit_dict['audit_template_uuid'] = ( '01234567-8910-1112-1314-151617181920') @@ -431,6 +433,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] with mock.patch.object(self.dbapi, 'create_audit', wraps=self.dbapi.create_audit) as cn_mock: response = self.post_json('/audits', audit_dict) @@ -447,6 +450,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict) self.assertEqual('application/json', response.content_type) @@ -462,6 +466,7 @@ class TestPost(api_base.FunctionalTest): audit_dict = post_get_test_audit() del audit_dict['uuid'] del audit_dict['state'] + del audit_dict['scope'] audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value audit_dict['interval'] = 1200 @@ -482,6 +487,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['state'] audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict, expect_errors=True) self.assertEqual(400, response.status_int) @@ -500,6 +506,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['state'] audit_dict['audit_type'] = objects.audit.AuditType.ONESHOT.value audit_dict['interval'] = 1200 + del audit_dict['scope'] response = self.post_json('/audits', audit_dict, expect_errors=True) self.assertEqual(400, response.status_int) @@ -515,6 +522,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict) de_mock.assert_called_once_with(mock.ANY, response.json['uuid']) @@ -523,6 +531,7 @@ class TestPost(api_base.FunctionalTest): mock_trigger_audit.return_value = mock.ANY audit_dict = post_get_test_audit(state=objects.audit.State.PENDING) + del audit_dict['scope'] response = self.post_json('/audits', audit_dict, expect_errors=True) self.assertEqual('application/json', response.content_type) self.assertEqual(400, response.status_int) @@ -536,6 +545,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict, expect_errors=True) self.assertEqual('application/json', response.content_type) @@ -556,6 +566,7 @@ class TestPost(api_base.FunctionalTest): del audit_dict['uuid'] del audit_dict['state'] del audit_dict['interval'] + del audit_dict['scope'] response = self.post_json('/audits', audit_dict, expect_errors=True) self.assertEqual('application/json', response.content_type) @@ -577,7 +588,8 @@ class TestPost(api_base.FunctionalTest): parameters={'fake1': 1, 'fake2': "hello"}) audit_dict['audit_template_uuid'] = audit_template['uuid'] - del_keys = ['uuid', 'goal_id', 'strategy_id', 'state', 'interval'] + del_keys = ['uuid', 'goal_id', 'strategy_id', 'state', 'interval', + 'scope'] for k in del_keys: del audit_dict[k] @@ -738,6 +750,7 @@ class TestAuaditPolicyEnforcement(api_base.FunctionalTest): audit_dict = post_get_test_audit(state=objects.audit.State.PENDING) del audit_dict['uuid'] del audit_dict['state'] + del audit_dict['scope'] self._common_policy_check( "audit:create", self.post_json, '/audits', audit_dict, expect_errors=True) diff --git a/watcher/tests/db/test_audit_template.py b/watcher/tests/db/test_audit_template.py index 013f4bb6c..b206fec5a 100644 --- a/watcher/tests/db/test_audit_template.py +++ b/watcher/tests/db/test_audit_template.py @@ -247,7 +247,6 @@ class DbAuditTemplateTestCase(base.DbTestCase): uuid=w_utils.generate_uuid(), name='My Audit Template 1', description='Description of my audit template 1', - host_aggregate=5, goal='DUMMY', extra={'automatic': True}) audit_template2 = self._create_test_audit_template( @@ -255,18 +254,9 @@ class DbAuditTemplateTestCase(base.DbTestCase): uuid=w_utils.generate_uuid(), name='My Audit Template 2', description='Description of my audit template 2', - host_aggregate=3, goal='DUMMY', extra={'automatic': True}) - res = self.dbapi.get_audit_template_list(self.context, - filters={'host_aggregate': 5}) - self.assertEqual([audit_template1['id']], [r.id for r in res]) - - res = self.dbapi.get_audit_template_list(self.context, - filters={'host_aggregate': 1}) - self.assertEqual([], [r.id for r in res]) - res = self.dbapi.get_audit_template_list( self.context, filters={'goal': 'DUMMY'}) diff --git a/watcher/tests/db/utils.py b/watcher/tests/db/utils.py index 160a5c471..319a8ca90 100644 --- a/watcher/tests/db/utils.py +++ b/watcher/tests/db/utils.py @@ -28,11 +28,11 @@ def get_test_audit_template(**kwargs): 'name': kwargs.get('name', 'My Audit Template'), 'description': kwargs.get('description', 'Desc. Of My Audit Template'), 'extra': kwargs.get('extra', {'automatic': False}), - 'host_aggregate': kwargs.get('host_aggregate', 1), 'version': kwargs.get('version', 'v1'), 'created_at': kwargs.get('created_at'), 'updated_at': kwargs.get('updated_at'), 'deleted_at': kwargs.get('deleted_at'), + 'scope': kwargs.get('scope', []), } @@ -66,7 +66,7 @@ def get_test_audit(**kwargs): 'interval': kwargs.get('period', 3600), 'goal_id': kwargs.get('goal_id', 1), 'strategy_id': kwargs.get('strategy_id', None), - 'host_aggregate': kwargs.get('host_aggregate', 1), + 'scope': kwargs.get('scope', []), } diff --git a/watcher/tests/decision_engine/audit/test_audit_handlers.py b/watcher/tests/decision_engine/audit/test_audit_handlers.py index 20210f23f..36db3596d 100644 --- a/watcher/tests/decision_engine/audit/test_audit_handlers.py +++ b/watcher/tests/decision_engine/audit/test_audit_handlers.py @@ -148,7 +148,7 @@ class TestContinuousAuditHandler(base.DbTestCase): audit_handler.launch_audits_periodically() mock_add_job.assert_has_calls(calls) - audit_handler.update_audit_state(self.context, audits[1], + audit_handler.update_audit_state(audits[1], audit_objects.State.CANCELLED) is_inactive = audit_handler._is_audit_inactive(audits[1]) self.assertTrue(is_inactive) diff --git a/watcher/tests/decision_engine/scope/__init__.py b/watcher/tests/decision_engine/scope/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/scope/fake_scopes.py b/watcher/tests/decision_engine/scope/fake_scopes.py new file mode 100644 index 000000000..9e638fe1d --- /dev/null +++ b/watcher/tests/decision_engine/scope/fake_scopes.py @@ -0,0 +1,35 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 Servionica +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +fake_scope_1 = [{'availability_zones': [{'name': 'AZ1'}]}, + {'exclude': + [{'instances': + [{'uuid': 'INSTANCE_6'}]}] + } + ] + +default_scope = [{'host_aggregates': [{'id': '*'}]}, + {'availability_zones': [{'name': 'AZ1'}, + {'name': 'AZ2'}]}, + {'exclude': [ + {'instances': [ + {'uuid': 'INSTANCE_1'}, + {'uuid': 'INSTANCE_2'}]}, + {'compute_nodes': [ + {'name': 'Node_1'}, + {'name': 'Node_2'}]} + ]} + ] diff --git a/watcher/tests/decision_engine/scope/test_default.py b/watcher/tests/decision_engine/scope/test_default.py new file mode 100644 index 000000000..2c31d187f --- /dev/null +++ b/watcher/tests/decision_engine/scope/test_default.py @@ -0,0 +1,210 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 Servionica +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from jsonschema import validators +import mock + +from watcher.common import exception +from watcher.common import nova_helper +from watcher.decision_engine.scope import default +from watcher.tests import base +from watcher.tests.decision_engine.model import faker_cluster_state +from watcher.tests.decision_engine.scope import fake_scopes + + +class TestDefaultScope(base.TestCase): + + def setUp(self): + super(TestDefaultScope, self).setUp() + self.fake_cluster = faker_cluster_state.FakerModelCollector() + + @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') + def test_get_scoped_model_with_zones_and_instances(self, mock_zone_list): + cluster = self.fake_cluster.generate_scenario_1() + audit_scope = fake_scopes.fake_scope_1 + mock_zone_list.return_value = [ + mock.Mock(zoneName='AZ{0}'.format(i), + hosts={'Node_{0}'.format(i): {}}) + for i in range(2)] + model = default.DefaultScope(audit_scope, + osc=mock.Mock()).get_scoped_model(cluster) + nodes = {'Node_4': set([]), 'Node_0': set([]), 'Node_3': set([]), + 'Node_1': set(['INSTANCE_2']), 'Node_2': set([])} + self.assertEqual(nodes, model.get_mapping().get_mapping()) + + @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') + def test_get_scoped_model_without_scope(self, mock_zone_list): + cluster = self.fake_cluster.generate_scenario_1() + default.DefaultScope([], + osc=mock.Mock()).get_scoped_model(cluster) + assert not mock_zone_list.called + + def test__remove_instance(self): + cluster = self.fake_cluster.generate_scenario_1() + default.DefaultScope([], + osc=mock.Mock())._remove_instance(cluster, + 'INSTANCE_2', + 'Node_1') + expected_map = {'Node_4': set(['INSTANCE_7']), 'Node_1': set([]), + 'Node_0': set(['INSTANCE_0', 'INSTANCE_1']), + 'Node_3': set(['INSTANCE_6']), + 'Node_2': set(['INSTANCE_4', 'INSTANCE_5', + 'INSTANCE_3'])} + self.assertEqual(expected_map, cluster.get_mapping().get_mapping()) + + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail') + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list') + def test__collect_aggregates(self, mock_aggregate, + mock_detailed_aggregate): + allowed_nodes = [] + mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)] + mock_detailed_aggregate.side_effect = [ + mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)] + default.DefaultScope([{'host_aggregates': [{'id': 1}, {'id': 2}]}], + osc=mock.Mock())._collect_aggregates( + [{'id': 1}, {'id': 2}], allowed_nodes) + self.assertEqual(['Node_1'], allowed_nodes) + + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail') + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list') + def test_aggregates_wildcard_is_used(self, mock_aggregate, + mock_detailed_aggregate): + allowed_nodes = [] + mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)] + mock_detailed_aggregate.side_effect = [ + mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) for i in range(2)] + default.DefaultScope([{'host_aggregates': [{'id': '*'}]}], + osc=mock.Mock())._collect_aggregates( + [{'id': '*'}], allowed_nodes) + self.assertEqual(['Node_0', 'Node_1'], allowed_nodes) + + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list') + def test_aggregates_wildcard_with_other_ids(self, mock_aggregate): + allowed_nodes = [] + mock_aggregate.return_value = [mock.Mock(id=i) for i in range(2)] + scope_handler = default.DefaultScope( + [{'host_aggregates': [{'id': '*'}, {'id': 1}]}], + osc=mock.Mock()) + self.assertRaises(exception.WildcardCharacterIsUsed, + scope_handler._collect_aggregates, + [{'id': '*'}, {'id': 1}], + allowed_nodes) + + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_detail') + @mock.patch.object(nova_helper.NovaHelper, 'get_aggregate_list') + def test_aggregates_with_names_and_ids(self, mock_aggregate, + mock_detailed_aggregate): + allowed_nodes = [] + mock_aggregate.return_value = [mock.Mock(id=i, + name="HA_{0}".format(i)) + for i in range(2)] + mock_collection = [mock.Mock(id=i, hosts=['Node_{0}'.format(i)]) + for i in range(2)] + mock_collection[0].name = 'HA_0' + mock_collection[1].name = 'HA_1' + + mock_detailed_aggregate.side_effect = mock_collection + + default.DefaultScope([{'host_aggregates': [{'name': 'HA_1'}, + {'id': 0}]}], + osc=mock.Mock())._collect_aggregates( + [{'name': 'HA_1'}, {'id': 0}], allowed_nodes) + self.assertEqual(['Node_0', 'Node_1'], allowed_nodes) + + @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') + def test__collect_zones(self, mock_zone_list): + allowed_nodes = [] + mock_zone_list.return_value = [ + mock.Mock(zoneName="AZ{0}".format(i+1), + hosts={'Node_{0}'.format(2*i): 1, + 'Node_{0}'.format(2*i+1): 2}) + for i in range(2)] + default.DefaultScope([{'availability_zones': [{'name': "AZ1"}]}], + osc=mock.Mock())._collect_zones( + [{'name': "AZ1"}], allowed_nodes) + self.assertEqual(['Node_0', 'Node_1'], sorted(allowed_nodes)) + + @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') + def test_zones_wildcard_is_used(self, mock_zone_list): + allowed_nodes = [] + mock_zone_list.return_value = [ + mock.Mock(zoneName="AZ{0}".format(i+1), + hosts={'Node_{0}'.format(2*i): 1, + 'Node_{0}'.format(2*i+1): 2}) + for i in range(2)] + default.DefaultScope([{'availability_zones': [{'name': "*"}]}], + osc=mock.Mock())._collect_zones( + [{'name': "*"}], allowed_nodes) + self.assertEqual(['Node_0', 'Node_1', 'Node_2', 'Node_3'], + sorted(allowed_nodes)) + + @mock.patch.object(nova_helper.NovaHelper, 'get_availability_zone_list') + def test_zones_wildcard_with_other_ids(self, mock_zone_list): + allowed_nodes = [] + mock_zone_list.return_value = [ + mock.Mock(zoneName="AZ{0}".format(i+1), + hosts={'Node_{0}'.format(2*i): 1, + 'Node_{0}'.format(2*i+1): 2}) + for i in range(2)] + scope_handler = default.DefaultScope( + [{'availability_zones': [{'name': "*"}, {'name': 'AZ1'}]}], + osc=mock.Mock()) + self.assertRaises(exception.WildcardCharacterIsUsed, + scope_handler._collect_zones, + [{'name': "*"}, {'name': 'AZ1'}], + allowed_nodes) + + def test_default_schema(self): + test_scope = fake_scopes.default_scope + validators.Draft4Validator( + default.DefaultScope.DEFAULT_SCHEMA).validate(test_scope) + + def test__exclude_resources(self): + resources_to_exclude = [{'instances': [{'uuid': 'INSTANCE_1'}, + {'uuid': 'INSTANCE_2'}]}, + {'compute_nodes': [{'name': 'Node_1'}, + {'name': 'Node_2'}]} + ] + instances_to_exclude = [] + nodes_to_exclude = [] + default.DefaultScope([], osc=mock.Mock())._exclude_resources( + resources_to_exclude, instances=instances_to_exclude, + nodes=nodes_to_exclude) + self.assertEqual(['Node_1', 'Node_2'], sorted(nodes_to_exclude)) + self.assertEqual(['INSTANCE_1', 'INSTANCE_2'], + sorted(instances_to_exclude)) + + def test__remove_node_from_model(self): + cluster = self.fake_cluster.generate_scenario_1() + default.DefaultScope([], osc=mock.Mock())._remove_node_from_model( + ['Node_1', 'Node_2'], cluster) + expected_cluster = {'Node_0': set(['INSTANCE_0', 'INSTANCE_1']), + 'Node_1': set([]), 'Node_2': set([]), + 'Node_3': set(['INSTANCE_6']), + 'Node_4': set(['INSTANCE_7'])} + self.assertEqual(expected_cluster, cluster.get_mapping().get_mapping()) + + def test__remove_instances_from_model(self): + cluster = self.fake_cluster.generate_scenario_1() + default.DefaultScope([], osc=mock.Mock())._remove_instances_from_model( + ['INSTANCE_1', 'INSTANCE_2'], cluster) + expected_cluster = {'Node_0': set(['INSTANCE_0']), 'Node_1': set([]), + 'Node_2': set(['INSTANCE_3', 'INSTANCE_4', + 'INSTANCE_5']), + 'Node_3': set(['INSTANCE_6']), + 'Node_4': set(['INSTANCE_7'])} + self.assertEqual(expected_cluster, cluster.get_mapping().get_mapping()) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py b/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py index f75ad0ce2..afbe5c4f4 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_basic_consolidation.py @@ -17,12 +17,12 @@ # limitations under the License. # import collections +import copy import mock from watcher.applier.loading import default from watcher.common import clients from watcher.common import exception -from watcher.decision_engine.model.collector import nova from watcher.decision_engine.model import model_root from watcher.decision_engine.strategy import strategies from watcher.tests import base @@ -45,7 +45,8 @@ class TestBasicConsolidation(base.TestCase): self.addCleanup(p_osc.stop) p_model = mock.patch.object( - nova.NovaClusterDataModelCollector, "execute") + strategies.BasicConsolidation, "compute_model", + new_callable=mock.PropertyMock) self.m_model = p_model.start() self.addCleanup(p_model.stop) @@ -55,6 +56,15 @@ class TestBasicConsolidation(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.BasicConsolidation, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_model.return_value = model_root.ModelRoot() self.m_ceilometer.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics) @@ -168,7 +178,7 @@ class TestBasicConsolidation(base.TestCase): def test_basic_consolidation_works_on_model_copy(self): model = self.fake_cluster.generate_scenario_3_with_2_nodes() - self.m_model.return_value = model + self.m_model.return_value = copy.deepcopy(model) self.assertEqual( model.to_string(), self.strategy.compute_model.to_string()) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_dummy_strategy.py b/watcher/tests/decision_engine/strategy/strategies/test_dummy_strategy.py index 57626c943..9ce0a7c50 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_dummy_strategy.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_dummy_strategy.py @@ -37,6 +37,15 @@ class TestDummyStrategy(base.TestCase): self.m_model = p_model.start() self.addCleanup(p_model.stop) + p_audit_scope = mock.patch.object( + strategies.DummyStrategy, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_model.return_value = model_root.ModelRoot() self.strategy = strategies.DummyStrategy(config=mock.Mock()) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_outlet_temp_control.py b/watcher/tests/decision_engine/strategy/strategies/test_outlet_temp_control.py index 3f21c4d3a..ab4a9273d 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_outlet_temp_control.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_outlet_temp_control.py @@ -51,6 +51,15 @@ class TestOutletTempControl(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.OutletTempControl, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_model.return_value = model_root.ModelRoot() self.m_ceilometer.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py b/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py index 9a1591e22..43d70e7bd 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_uniform_airflow.py @@ -51,6 +51,15 @@ class TestUniformAirflow(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.UniformAirflow, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_model.return_value = model_root.ModelRoot() self.m_ceilometer.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_vm_workload_consolidation.py b/watcher/tests/decision_engine/strategy/strategies/test_vm_workload_consolidation.py index c3be01d64..9cf329abf 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_vm_workload_consolidation.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_vm_workload_consolidation.py @@ -47,6 +47,15 @@ class TestVMWorkloadConsolidation(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.VMWorkloadConsolidation, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + # fake metrics self.fake_metrics = faker_cluster_and_metrics.FakeCeilometerMetrics( self.m_model.return_value) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_workload_balance.py b/watcher/tests/decision_engine/strategy/strategies/test_workload_balance.py index 59ab549a0..9260b6e7d 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_workload_balance.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_workload_balance.py @@ -51,6 +51,15 @@ class TestWorkloadBalance(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.WorkloadBalance, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_model.return_value = model_root.ModelRoot() self.m_ceilometer.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics_wb) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py b/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py index 16e4f79f2..6618266c3 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py @@ -57,9 +57,17 @@ class TestWorkloadStabilization(base.TestCase): self.m_ceilometer = p_ceilometer.start() self.addCleanup(p_ceilometer.stop) + p_audit_scope = mock.patch.object( + strategies.WorkloadStabilization, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + self.m_model.return_value = model_root.ModelRoot() self.m_ceilometer.return_value = mock.Mock( statistic_aggregation=self.fake_metrics.mock_get_statistics) + self.m_audit_scope.return_value = mock.Mock() self.strategy = strategies.WorkloadStabilization(config=mock.Mock()) self.strategy.input_parameters = utils.Struct() self.strategy.input_parameters.update( diff --git a/watcher_tempest_plugin/services/infra_optim/v1/json/client.py b/watcher_tempest_plugin/services/infra_optim/v1/json/client.py index be3446839..6ee5399df 100644 --- a/watcher_tempest_plugin/services/infra_optim/v1/json/client.py +++ b/watcher_tempest_plugin/services/infra_optim/v1/json/client.py @@ -62,8 +62,6 @@ class InfraOptimClientJSON(base.BaseInfraOptimClient): :param description: The description of the audit template. :param goal_uuid: The related Goal UUID associated. :param strategy_uuid: The related Strategy UUID associated. - :param host_aggregate: ID of the host aggregate targeted by - this audit template. :param extra: Metadata associated to this audit template. :return: A tuple with the server response and the created audit template. @@ -78,7 +76,6 @@ class InfraOptimClientJSON(base.BaseInfraOptimClient): 'description': parameters.get('description'), 'goal': parameters.get('goal'), 'strategy': parameters.get('strategy'), - 'host_aggregate': parameters.get('host_aggregate', 1), 'extra': parameters.get('extra', {}), } diff --git a/watcher_tempest_plugin/tests/api/admin/base.py b/watcher_tempest_plugin/tests/api/admin/base.py index 98935a564..2de745337 100644 --- a/watcher_tempest_plugin/tests/api/admin/base.py +++ b/watcher_tempest_plugin/tests/api/admin/base.py @@ -115,24 +115,21 @@ class BaseInfraOptimTest(test.BaseTestCase): @classmethod def create_audit_template(cls, goal, name=None, description=None, - strategy=None, host_aggregate=None, - extra=None): + strategy=None, extra=None): """Wrapper utility for creating a test audit template :param goal: Goal UUID or name related to the audit template. :param name: The name of the audit template. Default: My Audit Template :param description: The description of the audit template. :param strategy: Strategy UUID or name related to the audit template. - :param host_aggregate: ID of the host aggregate targeted by - this audit template. :param extra: Metadata associated to this audit template. :return: A tuple with The HTTP response and its body """ description = description or data_utils.rand_name( 'test-audit_template') resp, body = cls.client.create_audit_template( - name=name, description=description, goal=goal, strategy=strategy, - host_aggregate=host_aggregate, extra=extra) + name=name, description=description, goal=goal, + strategy=strategy, extra=extra) cls.created_audit_templates.add(body['uuid']) diff --git a/watcher_tempest_plugin/tests/api/admin/test_audit_template.py b/watcher_tempest_plugin/tests/api/admin/test_audit_template.py index 045ca1fc3..e3bc78372 100644 --- a/watcher_tempest_plugin/tests/api/admin/test_audit_template.py +++ b/watcher_tempest_plugin/tests/api/admin/test_audit_template.py @@ -35,7 +35,6 @@ class TestCreateDeleteAuditTemplate(base.BaseInfraOptimTest): params = { 'name': 'my at name %s' % uuid.uuid4(), 'description': 'my at description', - 'host_aggregate': 12, 'goal': goal['uuid'], 'extra': {'str': 'value', 'int': 123, 'float': 0.123, 'bool': True, 'list': [1, 2, 3], @@ -43,7 +42,6 @@ class TestCreateDeleteAuditTemplate(base.BaseInfraOptimTest): expected_data = { 'name': params['name'], 'description': params['description'], - 'host_aggregate': params['host_aggregate'], 'goal_uuid': params['goal'], 'goal_name': goal_name, 'strategy_uuid': None, @@ -64,14 +62,12 @@ class TestCreateDeleteAuditTemplate(base.BaseInfraOptimTest): params = { 'name': 'my at name %s' % uuid.uuid4(), 'description': 'my àt déscrïptïôn', - 'host_aggregate': 12, 'goal': goal['uuid'], 'extra': {'foo': 'bar'}} expected_data = { 'name': params['name'], 'description': params['description'], - 'host_aggregate': params['host_aggregate'], 'goal_uuid': params['goal'], 'goal_name': goal_name, 'strategy_uuid': None, @@ -170,7 +166,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): params = {'name': 'my at name %s' % uuid.uuid4(), 'description': 'my at description', - 'host_aggregate': 12, 'goal': self.goal['uuid'], 'extra': {'key1': 'value1', 'key2': 'value2'}} @@ -178,7 +173,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): new_name = 'my at new name %s' % uuid.uuid4() new_description = 'my new at description' - new_host_aggregate = 10 new_extra = {'key1': 'new-value1', 'key2': 'new-value2'} patch = [{'path': '/name', @@ -187,9 +181,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): {'path': '/description', 'op': 'replace', 'value': new_description}, - {'path': '/host_aggregate', - 'op': 'replace', - 'value': new_host_aggregate}, {'path': '/goal', 'op': 'replace', 'value': new_goal['uuid']}, @@ -208,7 +199,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): _, body = self.client.show_audit_template(body['uuid']) self.assertEqual(new_name, body['name']) self.assertEqual(new_description, body['description']) - self.assertEqual(new_host_aggregate, body['host_aggregate']) self.assertEqual(new_goal['uuid'], body['goal_uuid']) self.assertEqual(new_strategy['uuid'], body['strategy_uuid']) self.assertEqual(new_extra, body['extra']) @@ -220,7 +210,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): name = 'my at name %s' % uuid.uuid4() params = {'name': name, 'description': description, - 'host_aggregate': 12, 'goal': self.goal['uuid'], 'extra': extra} @@ -242,13 +231,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): _, body = self.client.show_audit_template(audit_template['uuid']) self.assertEqual({}, body['extra']) - # Removing the Host Aggregate ID - self.client.update_audit_template( - audit_template['uuid'], - [{'path': '/host_aggregate', 'op': 'remove'}]) - _, body = self.client.show_audit_template(audit_template['uuid']) - self.assertEqual({}, body['extra']) - # Assert nothing else was changed self.assertEqual(name, body['name']) self.assertEqual(description, body['description']) @@ -258,7 +240,6 @@ class TestAuditTemplate(base.BaseInfraOptimTest): def test_update_audit_template_add(self): params = {'name': 'my at name %s' % uuid.uuid4(), 'description': 'my at description', - 'host_aggregate': 12, 'goal': self.goal['uuid']} _, body = self.create_audit_template(**params) diff --git a/watcher_tempest_plugin/tests/scenario/base.py b/watcher_tempest_plugin/tests/scenario/base.py index e55def4bb..bbd67b9f7 100644 --- a/watcher_tempest_plugin/tests/scenario/base.py +++ b/watcher_tempest_plugin/tests/scenario/base.py @@ -74,24 +74,21 @@ class BaseInfraOptimScenarioTest(manager.ScenarioTest): # ### AUDIT TEMPLATES ### # def create_audit_template(self, goal, name=None, description=None, - strategy=None, host_aggregate=None, - extra=None): + strategy=None, extra=None): """Wrapper utility for creating a test audit template :param goal: Goal UUID or name related to the audit template. :param name: The name of the audit template. Default: My Audit Template :param description: The description of the audit template. :param strategy: Strategy UUID or name related to the audit template. - :param host_aggregate: ID of the host aggregate targeted by - this audit template. :param extra: Metadata associated to this audit template. :return: A tuple with The HTTP response and its body """ description = description or data_utils.rand_name( 'test-audit_template') resp, body = self.client.create_audit_template( - name=name, description=description, goal=goal, strategy=strategy, - host_aggregate=host_aggregate, extra=extra) + name=name, description=description, goal=goal, + strategy=strategy, extra=extra) self.addCleanup( self.delete_audit_template,