From 5a28ac772ac2e2c57f5d8acc23b61c21c9387e79 Mon Sep 17 00:00:00 2001 From: Yumeng Bao Date: Fri, 23 Jun 2017 11:35:37 +0800 Subject: [PATCH] Saving Energy Strategy Add strategy to trigger "power on" and "power off" actions in watcher. Change-Id: I7ebcd2a0282e3cc7b9b01cf8c744468ce16c56bb Implements: blueprint strategy-to-trigger-power-on-and-power-off-actions Co-Authored-By: licanwei --- setup.cfg | 2 + watcher/decision_engine/goal/__init__.py | 3 +- watcher/decision_engine/goal/goals.py | 24 ++ .../strategy/strategies/__init__.py | 5 +- .../strategy/strategies/base.py | 8 + .../strategy/strategies/saving_energy.py | 199 +++++++++++++++++ .../strategy/strategies/test_saving_energy.py | 210 ++++++++++++++++++ 7 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 watcher/decision_engine/strategy/strategies/saving_energy.py create mode 100644 watcher/tests/decision_engine/strategy/strategies/test_saving_energy.py diff --git a/setup.cfg b/setup.cfg index 7dc6122e1..e21534b49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,6 +54,7 @@ watcher_goals = workload_balancing = watcher.decision_engine.goal.goals:WorkloadBalancing airflow_optimization = watcher.decision_engine.goal.goals:AirflowOptimization noisy_neighbor = watcher.decision_engine.goal.goals:NoisyNeighborOptimization + saving_energy = watcher.decision_engine.goal.goals:SavingEnergy watcher_scoring_engines = dummy_scorer = watcher.decision_engine.scoring.dummy_scorer:DummyScorer @@ -67,6 +68,7 @@ watcher_strategies = dummy_with_resize = watcher.decision_engine.strategy.strategies.dummy_with_resize:DummyWithResize basic = watcher.decision_engine.strategy.strategies.basic_consolidation:BasicConsolidation outlet_temperature = watcher.decision_engine.strategy.strategies.outlet_temp_control:OutletTempControl + saving_energy = watcher.decision_engine.strategy.strategies.saving_energy:SavingEnergy vm_workload_consolidation = watcher.decision_engine.strategy.strategies.vm_workload_consolidation:VMWorkloadConsolidation workload_stabilization = watcher.decision_engine.strategy.strategies.workload_stabilization:WorkloadStabilization workload_balance = watcher.decision_engine.strategy.strategies.workload_balance:WorkloadBalance diff --git a/watcher/decision_engine/goal/__init__.py b/watcher/decision_engine/goal/__init__.py index 160788427..1cde9744c 100644 --- a/watcher/decision_engine/goal/__init__.py +++ b/watcher/decision_engine/goal/__init__.py @@ -22,7 +22,8 @@ ThermalOptimization = goals.ThermalOptimization Unclassified = goals.Unclassified WorkloadBalancing = goals.WorkloadBalancing NoisyNeighbor = goals.NoisyNeighborOptimization +SavingEnergy = goals.SavingEnergy __all__ = ("Dummy", "ServerConsolidation", "ThermalOptimization", "Unclassified", "WorkloadBalancing", - "NoisyNeighborOptimization",) + "NoisyNeighborOptimization", "SavingEnergy") diff --git a/watcher/decision_engine/goal/goals.py b/watcher/decision_engine/goal/goals.py index e5be78fc4..58909dde5 100644 --- a/watcher/decision_engine/goal/goals.py +++ b/watcher/decision_engine/goal/goals.py @@ -192,3 +192,27 @@ class NoisyNeighborOptimization(base.Goal): def get_efficacy_specification(cls): """The efficacy spec for the current goal""" return specs.Unclassified() + + +class SavingEnergy(base.Goal): + """SavingEnergy + + This goal is used to reduce power consumption within a data center. + """ + + @classmethod + def get_name(cls): + return "saving_energy" + + @classmethod + def get_display_name(cls): + return _("Saving Energy") + + @classmethod + def get_translatable_display_name(cls): + return "Saving Energy" + + @classmethod + def get_efficacy_specification(cls): + """The efficacy spec for the current goal""" + return specs.Unclassified() diff --git a/watcher/decision_engine/strategy/strategies/__init__.py b/watcher/decision_engine/strategy/strategies/__init__.py index c1a28217a..afefced2c 100644 --- a/watcher/decision_engine/strategy/strategies/__init__.py +++ b/watcher/decision_engine/strategy/strategies/__init__.py @@ -19,6 +19,7 @@ from watcher.decision_engine.strategy.strategies import dummy_strategy from watcher.decision_engine.strategy.strategies import dummy_with_scorer from watcher.decision_engine.strategy.strategies import noisy_neighbor from watcher.decision_engine.strategy.strategies import outlet_temp_control +from watcher.decision_engine.strategy.strategies import saving_energy from watcher.decision_engine.strategy.strategies import uniform_airflow from watcher.decision_engine.strategy.strategies import \ vm_workload_consolidation @@ -29,6 +30,7 @@ BasicConsolidation = basic_consolidation.BasicConsolidation OutletTempControl = outlet_temp_control.OutletTempControl DummyStrategy = dummy_strategy.DummyStrategy DummyWithScorer = dummy_with_scorer.DummyWithScorer +SavingEnergy = saving_energy.SavingEnergy VMWorkloadConsolidation = vm_workload_consolidation.VMWorkloadConsolidation WorkloadBalance = workload_balance.WorkloadBalance WorkloadStabilization = workload_stabilization.WorkloadStabilization @@ -37,4 +39,5 @@ NoisyNeighbor = noisy_neighbor.NoisyNeighbor __all__ = ("BasicConsolidation", "OutletTempControl", "DummyStrategy", "DummyWithScorer", "VMWorkloadConsolidation", "WorkloadBalance", - "WorkloadStabilization", "UniformAirflow", "NoisyNeighbor") + "WorkloadStabilization", "UniformAirflow", "NoisyNeighbor", + "SavingEnergy") diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index 607f98a37..406baa75b 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -358,3 +358,11 @@ class NoisyNeighborBaseStrategy(BaseStrategy): @classmethod def get_goal_name(cls): return "noisy_neighbor" + + +@six.add_metaclass(abc.ABCMeta) +class SavingEnergyBaseStrategy(BaseStrategy): + + @classmethod + def get_goal_name(cls): + return "saving_energy" diff --git a/watcher/decision_engine/strategy/strategies/saving_energy.py b/watcher/decision_engine/strategy/strategies/saving_energy.py new file mode 100644 index 000000000..f31b90182 --- /dev/null +++ b/watcher/decision_engine/strategy/strategies/saving_energy.py @@ -0,0 +1,199 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE Corporation +# +# Authors: licanwei +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import random + +from oslo_log import log + +from watcher._i18n import _ +from watcher.common import exception as wexc +from watcher.decision_engine.strategy.strategies import base + +LOG = log.getLogger(__name__) + + +class SavingEnergy(base.SavingEnergyBaseStrategy): + + def __init__(self, config, osc=None): + + super(SavingEnergy, self).__init__(config, osc) + self._ironic_client = None + self._nova_client = None + + self.with_vms_node_pool = [] + self.free_poweron_node_pool = [] + self.free_poweroff_node_pool = [] + self.free_used_percent = 0 + self.min_free_hosts_num = 1 + + @property + def ironic_client(self): + if not self._ironic_client: + self._ironic_client = self.osc.ironic() + return self._ironic_client + + @property + def nova_client(self): + if not self._nova_client: + self._nova_client = self.osc.nova() + return self._nova_client + + @classmethod + def get_name(cls): + return "saving_energy" + + @classmethod + def get_display_name(cls): + return _("Saving Energy Strategy") + + @classmethod + def get_translatable_display_name(cls): + return "Saving Energy Strategy" + + @classmethod + def get_schema(cls): + """return a schema of two input parameters + + The standby nodes refer to those nodes unused + but still poweredon to deal with boom of new instances. + """ + return { + "properties": { + "free_used_percent": { + "description": ("a rational number, which describes the" + "quotient of" + " min_free_hosts_num/nodes_with_VMs_num" + "where nodes_with_VMs_num is the number" + "of nodes with VMs"), + "type": "number", + "default": 10.0 + }, + "min_free_hosts_num": { + "description": ("minimum number of hosts without VMs" + "but still powered on"), + "type": "number", + "default": 1 + }, + }, + } + + def add_action_poweronoff_node(self, node_uuid, state): + """Add an action for node disability into the solution. + + :param node: node uuid + :param state: node power state, power on or power off + :return: None + """ + params = {'state': state} + self.solution.add_action( + action_type='change_node_power_state', + resource_id=node_uuid, + input_parameters=params) + + def get_hosts_pool(self): + """Get three pools, with_vms_node_pool, free_poweron_node_pool, + + free_poweroff_node_pool. + + """ + + node_list = self.ironic_client.node.list() + for node in node_list: + node_uuid = (node.to_dict())['uuid'] + node_info = self.ironic_client.node.get(node_uuid).to_dict() + hypervisor_id = node_info['extra'].get('compute_node_id', None) + if hypervisor_id is None: + LOG.warning(('Cannot find compute_node_id in extra ' + 'of ironic node %s'), node_uuid) + continue + hypervisor_node = self.nova_client.hypervisors.get(hypervisor_id) + if hypervisor_node is None: + LOG.warning(('Cannot find hypervisor %s'), hypervisor_id) + continue + hypervisor_node = hypervisor_node.to_dict() + compute_service = hypervisor_node.get('service', None) + host_uuid = compute_service.get('host') + if not self.compute_model.get_node_by_uuid(host_uuid): + continue + + if not (hypervisor_node.get('state') == 'up'): + """filter nodes that are not in 'up' state""" + continue + else: + if (hypervisor_node['running_vms'] == 0): + if (node_info['power_state'] == 'power on'): + self.free_poweron_node_pool.append(node_uuid) + elif (node_info['power_state'] == 'power off'): + self.free_poweroff_node_pool.append(node_uuid) + else: + self.with_vms_node_pool.append(node_uuid) + + def save_energy(self): + + need_poweron = max( + (len(self.with_vms_node_pool) * self.free_used_percent / 100), ( + self.min_free_hosts_num)) + len_poweron = len(self.free_poweron_node_pool) + len_poweroff = len(self.free_poweroff_node_pool) + if len_poweron > need_poweron: + for node in random.sample(self.free_poweron_node_pool, + (len_poweron - need_poweron)): + self.add_action_poweronoff_node(node, 'off') + LOG.debug("power off %s", node) + elif len_poweron < need_poweron: + diff = need_poweron - len_poweron + for node in random.sample(self.free_poweroff_node_pool, + min(len_poweroff, diff)): + self.add_action_poweronoff_node(node, 'on') + LOG.debug("power on %s", node) + + def pre_execute(self): + """Pre-execution phase + + This can be used to fetch some pre-requisites or data. + """ + LOG.info("Initializing Saving Energy Strategy") + + if not self.compute_model: + raise wexc.ClusterStateNotDefined() + + if self.compute_model.stale: + raise wexc.ClusterStateStale() + + LOG.debug(self.compute_model.to_string()) + + def do_execute(self): + """Strategy execution phase + + This phase is where you should put the main logic of your strategy. + """ + self.free_used_percent = self.input_parameters.free_used_percent + self.min_free_hosts_num = self.input_parameters.min_free_hosts_num + + self.get_hosts_pool() + self.save_energy() + + def post_execute(self): + """Post-execution phase + + This can be used to compute the global efficacy + """ + self.solution.model = self.compute_model + + LOG.debug(self.compute_model.to_string()) diff --git a/watcher/tests/decision_engine/strategy/strategies/test_saving_energy.py b/watcher/tests/decision_engine/strategy/strategies/test_saving_energy.py new file mode 100644 index 000000000..df74fbda3 --- /dev/null +++ b/watcher/tests/decision_engine/strategy/strategies/test_saving_energy.py @@ -0,0 +1,210 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import mock + +from watcher.common import clients +from watcher.common import utils +from watcher.decision_engine.strategy import strategies +from watcher.tests import base +from watcher.tests.decision_engine.model import faker_cluster_and_metrics + + +class TestSavingEnergy(base.TestCase): + + def setUp(self): + super(TestSavingEnergy, self).setUp() + + mock_node1 = mock.Mock() + mock_node2 = mock.Mock() + mock_node1.to_dict.return_value = { + 'uuid': '922d4762-0bc5-4b30-9cb9-48ab644dd861'} + mock_node2.to_dict.return_value = { + 'uuid': '922d4762-0bc5-4b30-9cb9-48ab644dd862'} + self.fake_nodes = [mock_node1, mock_node2] + + # fake cluster + self.fake_cluster = faker_cluster_and_metrics.FakerModelCollector() + + p_model = mock.patch.object( + strategies.SavingEnergy, "compute_model", + new_callable=mock.PropertyMock) + self.m_model = p_model.start() + self.addCleanup(p_model.stop) + + p_ironic = mock.patch.object( + clients.OpenStackClients, 'ironic') + self.m_ironic = p_ironic.start() + self.addCleanup(p_ironic.stop) + + p_nova = mock.patch.object( + clients.OpenStackClients, 'nova') + self.m_nova = p_nova.start() + self.addCleanup(p_nova.stop) + + p_model = mock.patch.object( + strategies.SavingEnergy, "compute_model", + new_callable=mock.PropertyMock) + self.m_model = p_model.start() + self.addCleanup(p_model.stop) + + p_audit_scope = mock.patch.object( + strategies.SavingEnergy, "audit_scope", + new_callable=mock.PropertyMock + ) + self.m_audit_scope = p_audit_scope.start() + self.addCleanup(p_audit_scope.stop) + + self.m_audit_scope.return_value = mock.Mock() + self.m_ironic.node.list.return_value = self.fake_nodes + + self.strategy = strategies.SavingEnergy( + config=mock.Mock()) + self.strategy.input_parameters = utils.Struct() + self.strategy.input_parameters.update( + {'free_used_percent': 10.0, + 'min_free_hosts_num': 1}) + self.strategy.free_used_percent = 10.0 + self.strategy.min_free_hosts_num = 1 + self.strategy._ironic_client = self.m_ironic + self.strategy._nova_client = self.m_nova + + def test_get_hosts_pool_with_vms_node_pool(self): + mock_node1 = mock.Mock() + mock_node2 = mock.Mock() + mock_node1.to_dict.return_value = { + 'extra': {'compute_node_id': 1}, + 'power_state': 'power on'} + mock_node2.to_dict.return_value = { + 'extra': {'compute_node_id': 2}, + 'power_state': 'power off'} + self.m_ironic.node.get.side_effect = [mock_node1, mock_node2] + + mock_hyper1 = mock.Mock() + mock_hyper2 = mock.Mock() + mock_hyper1.to_dict.return_value = { + 'running_vms': 2, 'service': {'host': 'Node_0'}, 'state': 'up'} + mock_hyper2.to_dict.return_value = { + 'running_vms': 2, 'service': {'host': 'Node_1'}, 'state': 'up'} + self.m_nova.hypervisors.get.side_effect = [mock_hyper1, mock_hyper2] + + model = self.fake_cluster.generate_scenario_1() + self.m_model.return_value = model + self.strategy.get_hosts_pool() + + self.assertEqual(len(self.strategy.with_vms_node_pool), 2) + self.assertEqual(len(self.strategy.free_poweron_node_pool), 0) + self.assertEqual(len(self.strategy.free_poweroff_node_pool), 0) + + def test_get_hosts_pool_free_poweron_node_pool(self): + mock_node1 = mock.Mock() + mock_node2 = mock.Mock() + mock_node1.to_dict.return_value = { + 'extra': {'compute_node_id': 1}, + 'power_state': 'power on'} + mock_node2.to_dict.return_value = { + 'extra': {'compute_node_id': 2}, + 'power_state': 'power on'} + self.m_ironic.node.get.side_effect = [mock_node1, mock_node2] + + mock_hyper1 = mock.Mock() + mock_hyper2 = mock.Mock() + mock_hyper1.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_0'}, 'state': 'up'} + mock_hyper2.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_1'}, 'state': 'up'} + self.m_nova.hypervisors.get.side_effect = [mock_hyper1, mock_hyper2] + + model = self.fake_cluster.generate_scenario_1() + self.m_model.return_value = model + self.strategy.get_hosts_pool() + + self.assertEqual(len(self.strategy.with_vms_node_pool), 0) + self.assertEqual(len(self.strategy.free_poweron_node_pool), 2) + self.assertEqual(len(self.strategy.free_poweroff_node_pool), 0) + + def test_get_hosts_pool_free_poweroff_node_pool(self): + mock_node1 = mock.Mock() + mock_node2 = mock.Mock() + mock_node1.to_dict.return_value = { + 'extra': {'compute_node_id': 1}, + 'power_state': 'power off'} + mock_node2.to_dict.return_value = { + 'extra': {'compute_node_id': 2}, + 'power_state': 'power off'} + self.m_ironic.node.get.side_effect = [mock_node1, mock_node2] + + mock_hyper1 = mock.Mock() + mock_hyper2 = mock.Mock() + mock_hyper1.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_0'}, 'state': 'up'} + mock_hyper2.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_1'}, 'state': 'up'} + self.m_nova.hypervisors.get.side_effect = [mock_hyper1, mock_hyper2] + + model = self.fake_cluster.generate_scenario_1() + self.m_model.return_value = model + self.strategy.get_hosts_pool() + + self.assertEqual(len(self.strategy.with_vms_node_pool), 0) + self.assertEqual(len(self.strategy.free_poweron_node_pool), 0) + self.assertEqual(len(self.strategy.free_poweroff_node_pool), 2) + + def test_save_energy_poweron(self): + self.strategy.free_poweroff_node_pool = [ + '922d4762-0bc5-4b30-9cb9-48ab644dd861', + '922d4762-0bc5-4b30-9cb9-48ab644dd862' + ] + self.strategy.save_energy() + self.assertEqual(len(self.strategy.solution.actions), 1) + action = self.strategy.solution.actions[0] + self.assertEqual(action.get('input_parameters').get('state'), 'on') + + def test_save_energy_poweroff(self): + self.strategy.free_poweron_node_pool = [ + '922d4762-0bc5-4b30-9cb9-48ab644dd861', + '922d4762-0bc5-4b30-9cb9-48ab644dd862' + ] + self.strategy.save_energy() + self.assertEqual(len(self.strategy.solution.actions), 1) + action = self.strategy.solution.actions[0] + self.assertEqual(action.get('input_parameters').get('state'), 'off') + + def test_execute(self): + mock_node1 = mock.Mock() + mock_node2 = mock.Mock() + mock_node1.to_dict.return_value = { + 'extra': {'compute_node_id': 1}, + 'power_state': 'power on'} + mock_node2.to_dict.return_value = { + 'extra': {'compute_node_id': 2}, + 'power_state': 'power on'} + self.m_ironic.node.get.side_effect = [mock_node1, mock_node2] + + mock_hyper1 = mock.Mock() + mock_hyper2 = mock.Mock() + mock_hyper1.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_0'}, 'state': 'up'} + mock_hyper2.to_dict.return_value = { + 'running_vms': 0, 'service': {'host': 'Node_1'}, 'state': 'up'} + self.m_nova.hypervisors.get.side_effect = [mock_hyper1, mock_hyper2] + + model = self.fake_cluster.generate_scenario_1() + self.m_model.return_value = model + + solution = self.strategy.execute() + self.assertEqual(len(solution.actions), 1)