diff --git a/doc/source/conf.py b/doc/source/conf.py index 129eea81d..88aa0e956 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,12 +19,13 @@ from watcher import version as watcher_version # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'oslo_config.sphinxconfiggen', + 'oslosphinx', 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinxcontrib.httpdomain', 'sphinxcontrib.pecanwsme.rest', + 'stevedore.sphinxext', 'wsmeext.sphinxext', - 'oslosphinx', 'watcher.doc', ] diff --git a/doc/source/dev/plugin/action-plugin.rst b/doc/source/dev/plugin/action-plugin.rst index 62883c30d..7969c2239 100644 --- a/doc/source/dev/plugin/action-plugin.rst +++ b/doc/source/dev/plugin/action-plugin.rst @@ -80,12 +80,9 @@ Here is an example showing how you can write a plugin called ``DummyAction``: pass -This implementation is the most basic one. So if you want to have more advanced -examples, have a look at the implementation of the actions already provided -by Watcher like. -To get a better understanding on how to implement a more advanced action, -have a look at the :py:class:`~watcher.applier.actions.migration.Migrate` -class. +This implementation is the most basic one. So in order to get a better +understanding on how to implement a more advanced action, have a look at the +:py:class:`~watcher.applier.actions.migration.Migrate` class. Input validation ---------------- diff --git a/doc/source/dev/plugin/base-setup.rst b/doc/source/dev/plugin/base-setup.rst index 37755ad05..3e829f562 100644 --- a/doc/source/dev/plugin/base-setup.rst +++ b/doc/source/dev/plugin/base-setup.rst @@ -88,10 +88,13 @@ Now that the project skeleton has been created, you can start the implementation of your plugin. As of now, you can implement the following plugins for Watcher: +- A :ref:`goal plugin ` - A :ref:`strategy plugin ` -- A :ref:`planner plugin ` - An :ref:`action plugin ` -- A :ref:`workflow engine plugin ` +- A :ref:`planner plugin ` +- A workflow engine plugin +- A :ref:`cluster data model collector plugin + ` If you want to learn more on how to implement them, you can refer to their dedicated documentation. diff --git a/doc/source/dev/plugin/cdmc-plugin.rst b/doc/source/dev/plugin/cdmc-plugin.rst new file mode 100644 index 000000000..3d229801d --- /dev/null +++ b/doc/source/dev/plugin/cdmc-plugin.rst @@ -0,0 +1,150 @@ +.. + Except where otherwise noted, this document is licensed under Creative + Commons Attribution 3.0 License. You can view the license at: + + https://creativecommons.org/licenses/by/3.0/ + +.. _implement_cluster_data_model_collector_plugin: + +======================================== +Build a new cluster data model collector +======================================== + +Watcher Decision Engine has an external cluster data model plugin interface +which gives anyone the ability to integrate an external cluster data model +collector in order to extend the initial set of cluster data model collectors +Watcher provides. + +This section gives some guidelines on how to implement and integrate custom +cluster data model collectors within Watcher. + + +Creating a new plugin +===================== + +First of all, you have to extend the :class:`~.BaseClusterDataModelCollector` +base class which defines the :py:meth:`~.BaseClusterDataModelCollector.execute` +abstract method you will have to implement. This method is responsible for +building an entire cluster data model. + +Here is an example showing how you can write a plugin called +``DummyClusterDataModelCollector``: + +.. code-block:: python + + # Filepath = /thirdparty/dummy.py + # Import path = thirdparty.dummy + + from watcher.decision_engine.model import model_root + from watcher.metrics_engine.cluster_model_collector import base + + + class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector): + + def execute(self): + model = model_root.ModelRoot() + # Do something here... + return model + +This implementation is the most basic one. So in order to get a better +understanding on how to implement a more advanced cluster data model collector, +have a look at the :py:class:`~.NovaClusterDataModelCollector` class. + +Define configuration parameters +=============================== + +At this point, you have a fully functional cluster data model collector. +However, in more complex implementation, you may want to define some +configuration options so one can tune the cluster data model collector to its +needs. To do so, you can implement the :py:meth:`~.Loadable.get_config_opts` +class method as followed: + +.. code-block:: python + + from oslo_config import cfg + from watcher.decision_engine.model import model_root + from watcher.metrics_engine.cluster_model_collector import base + + + class DummyClusterDataModelCollector(base.BaseClusterDataModelCollector): + + def execute(self): + model = model_root.ModelRoot() + # Do something here... + return model + + def get_config_opts(self): + return [ + cfg.StrOpt('test_opt', help="Demo Option.", default=0), + # Some more options ... + ] + +The configuration options defined within this class method will be included +within the global ``watcher.conf`` configuration file under a section named by +convention: ``{namespace}.{plugin_name}``. In our case, the ``watcher.conf`` +configuration would have to be modified as followed: + +.. code-block:: ini + + [watcher_cluster_data_model_collectors.dummy] + # Option used for testing. + test_opt = test_value + +Then, the configuration options you define within this method will then be +injected in each instantiated object via the ``config`` parameter of the +:py:meth:`~.BaseClusterDataModelCollector.__init__` method. + + +Abstract Plugin Class +===================== + +Here below is the abstract ``BaseClusterDataModelCollector`` class that every +single cluster data model collector should implement: + +.. autoclass:: watcher.metrics_engine.cluster_model_collector.base.BaseClusterDataModelCollector + :members: + :special-members: __init__ + :noindex: + + +Register a new entry point +========================== + +In order for the Watcher Decision Engine to load your new cluster data model +collector, the latter must be registered as a named entry point under the +``watcher_cluster_data_model_collectors`` entry point of your ``setup.py`` +file. If you are using pbr_, this entry point should be placed in your +``setup.cfg`` file. + +The name you give to your entry point has to be unique. + +Here below is how to register ``DummyClusterDataModelCollector`` using pbr_: + +.. code-block:: ini + + [entry_points] + watcher_cluster_data_model_collectors = + dummy = thirdparty.dummy:DummyClusterDataModelCollector + +.. _pbr: http://docs.openstack.org/developer/pbr/ + + +Using cluster data model collector plugins +========================================== + +The Watcher Decision Engine service will automatically discover any installed +plugins when it is restarted. If a Python package containing a custom plugin is +installed within the same environment as Watcher, Watcher will automatically +make that plugin available for use. + +At this point, you can use your new cluster data model plugin in your +:ref:`strategy plugin ` by using the +:py:attr:`~.BaseStrategy.collector_manager` property as followed: + +.. code-block:: python + + # [...] + dummy_collector = self.collector_manager.get_cluster_model_collector( + "dummy") # "dummy" is the name of the entry point we declared earlier + dummy_model = collector.get_latest_cluster_data_model() + # Do some stuff with this model diff --git a/doc/source/dev/plugins.rst b/doc/source/dev/plugins.rst index 4be2c53d9..422e84431 100644 --- a/doc/source/dev/plugins.rst +++ b/doc/source/dev/plugins.rst @@ -18,32 +18,43 @@ use the :ref:`Guru Meditation Reports ` to display them. Goals ===== -.. drivers-doc:: watcher_goals +.. list-plugins:: watcher_goals + :detailed: .. _watcher_strategies: Strategies ========== -.. drivers-doc:: watcher_strategies +.. list-plugins:: watcher_strategies + :detailed: .. _watcher_actions: Actions ======= -.. drivers-doc:: watcher_actions +.. list-plugins:: watcher_actions + :detailed: .. _watcher_workflow_engines: Workflow Engines ================ -.. drivers-doc:: watcher_workflow_engines +.. list-plugins:: watcher_workflow_engines + :detailed: .. _watcher_planners: Planners ======== -.. drivers-doc:: watcher_planners +.. list-plugins:: watcher_planners + :detailed: + +Cluster Data Model Collectors +============================= + +.. list-plugins:: watcher_cluster_data_model_collectors + :detailed: diff --git a/doc/source/index.rst b/doc/source/index.rst index 6b231724b..53677447f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -75,6 +75,7 @@ Plugins dev/plugin/base-setup dev/plugin/goal-plugin dev/plugin/strategy-plugin + dev/plugin/cdmc-plugin dev/plugin/action-plugin dev/plugin/planner-plugin dev/plugins diff --git a/watcher/decision_engine/goal/goals.py b/watcher/decision_engine/goal/goals.py index 4ce9dbda3..dc283b732 100644 --- a/watcher/decision_engine/goal/goals.py +++ b/watcher/decision_engine/goal/goals.py @@ -72,7 +72,7 @@ class Unclassified(base.Goal): class ServerConsolidation(base.Goal): - """ServerConsolidation + """Server Consolidation This goal is for efficient usage of compute server resources in order to reduce the total number of servers. @@ -97,7 +97,7 @@ class ServerConsolidation(base.Goal): class ThermalOptimization(base.Goal): - """ThermalOptimization + """Thermal Optimization This goal is used to balance the temperature across different servers. """ @@ -121,7 +121,7 @@ class ThermalOptimization(base.Goal): class WorkloadBalancing(base.Goal): - """WorkloadBalancing + """Workload Balancing This goal is used to evenly distribute workloads across different servers. """ @@ -145,6 +145,10 @@ class WorkloadBalancing(base.Goal): class AirflowOptimization(base.Goal): + """Workload Balancing + + This goal is used to optimize the air flow within a cloud infrastructure. + """ @classmethod def get_name(cls): diff --git a/watcher/decision_engine/model/mapping.py b/watcher/decision_engine/model/mapping.py index 72d3c2441..9d90e6762 100644 --- a/watcher/decision_engine/model/mapping.py +++ b/watcher/decision_engine/model/mapping.py @@ -30,7 +30,7 @@ class Mapping(object): self.lock = threading.Lock() def map(self, hypervisor, vm): - """Select the hypervisor where the instance are launched + """Select the hypervisor where the instance is launched :param hypervisor: the hypervisor :param vm: the virtual machine or instance diff --git a/watcher/doc.py b/watcher/doc.py index 96287d65f..63b096a89 100644 --- a/watcher/doc.py +++ b/watcher/doc.py @@ -22,7 +22,6 @@ import inspect from docutils import nodes from docutils.parsers import rst from docutils import statemachine -from stevedore import extension from watcher.version import version_info @@ -98,74 +97,6 @@ class WatcherTerm(BaseWatcherDirective): return node.children -class DriversDoc(BaseWatcherDirective): - """Directive to import an RST formatted docstring into the Watcher doc - - This directive imports the RST formatted docstring of every driver declared - within an entry point namespace provided as argument - - **How to use it** - - # inside your .py file - class DocumentedClassReferencedInEntrypoint(object): - '''My *.rst* docstring''' - - def foo(self): - '''Foo docstring''' - - # Inside your .rst file - .. drivers-doc:: entrypoint_namespace - :append_methods_doc: foo - - This directive will then import the docstring and then interprete it. - - Note that no section/sub-section can be imported via this directive as it - is a Sphinx restriction. - """ - - # You need to put an import path as an argument for this directive to work - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - has_content = False - - option_spec = dict( - # CSV formatted list of method names whose return values will be zipped - # together in the given order - append_methods_doc=lambda opts: [ - opt.strip() for opt in opts.split(",") if opt.strip()], - # By default, we always start by adding the driver object docstring - exclude_driver_docstring=rst.directives.flag, - ) - - def run(self): - ext_manager = extension.ExtensionManager(namespace=self.arguments[0]) - extensions = ext_manager.extensions - # Aggregates drivers based on their module name (i.e import path) - classes = [(ext.name, ext.plugin) for ext in extensions] - - for name, cls in classes: - self.add_line(".. rubric:: %s" % name) - self.add_line("") - - if "exclude_driver_docstring" not in self.options: - self.add_object_docstring(cls) - self.add_line("") - - for method_name in self.options.get("append_methods_doc", []): - if hasattr(cls, method_name): - method = getattr(cls, method_name) - method_result = inspect.cleandoc(method) - self.add_textblock(method_result()) - self.add_line("") - - node = nodes.paragraph() - node.document = self.state.document - self.state.nested_parse(self.result, 0, node) - return node.children - - def setup(app): - app.add_directive('drivers-doc', DriversDoc) app.add_directive('watcher-term', WatcherTerm) return {'version': version_info.version_string()} diff --git a/watcher/metrics_engine/cluster_model_collector/base.py b/watcher/metrics_engine/cluster_model_collector/base.py index 46ed09af0..1f84dcfbc 100644 --- a/watcher/metrics_engine/cluster_model_collector/base.py +++ b/watcher/metrics_engine/cluster_model_collector/base.py @@ -2,6 +2,7 @@ # Copyright (c) 2015 b<>com # # Authors: Jean-Emile DARTOIS +# Vincent FRANCOISE # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,93 +20,85 @@ """ A :ref:`Cluster Data Model ` is a logical -representation of the current state and topology of the -:ref:`Cluster ` -:ref:`Managed resources `. +representation of the current state and topology of the :ref:`Cluster +` :ref:`Managed resources `. -It is represented as a set of -:ref:`Managed resources ` -(which may be a simple tree or a flat list of key-value pairs) -which enables Watcher :ref:`Strategies ` to know the -current relationships between the different -:ref:`resources `) of the -:ref:`Cluster ` during an :ref:`Audit ` -and enables the :ref:`Strategy ` to request information -such as: +It is represented as a set of :ref:`Managed resources +` (which may be a simple tree or a flat list of +key-value pairs) which enables Watcher :ref:`Strategies ` +to know the current relationships between the different :ref:`resources +`) of the :ref:`Cluster ` +during an :ref:`Audit ` and enables the :ref:`Strategy +` to request information such as: -- What compute nodes are in a given - :ref:`Availability Zone ` - or a given :ref:`Host Aggregate ` ? -- What :ref:`Instances ` are hosted on a given compute - node ? -- What is the current load of a compute node ? -- What is the current free memory of a compute node ? -- What is the network link between two compute nodes ? -- What is the available bandwidth on a given network link ? -- What is the current space available on a given virtual disk of a given - :ref:`Instance ` ? -- What is the current state of a given :ref:`Instance `? -- ... +- What compute nodes are in a given :ref:`Availability Zone + ` or a given :ref:`Host Aggregate + `? +- What :ref:`Instances ` are hosted on a given compute + node? +- What is the current load of a compute node? +- What is the current free memory of a compute node? +- What is the network link between two compute nodes? +- What is the available bandwidth on a given network link? +- What is the current space available on a given virtual disk of a given + :ref:`Instance ` ? +- What is the current state of a given :ref:`Instance `? +- ... In a word, this data model enables the :ref:`Strategy ` to know: -- the current topology of the :ref:`Cluster ` -- the current capacity for each - :ref:`Managed resource ` -- the current amount of used/free space for each - :ref:`Managed resource ` -- the current state of each - :ref:`Managed resources ` +- the current topology of the :ref:`Cluster ` +- the current capacity for each :ref:`Managed resource + ` +- the current amount of used/free space for each :ref:`Managed resource + ` +- the current state of each :ref:`Managed resources + ` -In the Watcher project, we aim at providing a generic and very basic -:ref:`Cluster Data Model ` for each -:ref:`Goal `, usable in the associated -:ref:`Strategies ` through some helper classes in order -to: +In the Watcher project, we aim at providing a some generic and basic +:ref:`Cluster Data Model ` for each :ref:`Goal +`, usable in the associated :ref:`Strategies +` through a plugin-based mechanism that are directly +accessible from the strategies classes in order to: -- simplify the development of a new - :ref:`Strategy ` for a given - :ref:`Goal ` when there already are some existing - :ref:`Strategies ` associated to the same - :ref:`Goal ` -- avoid duplicating the same code in several - :ref:`Strategies ` associated to the same - :ref:`Goal ` -- have a better consistency between the different - :ref:`Strategies ` for a given - :ref:`Goal ` -- avoid any strong coupling with any external - :ref:`Cluster Data Model ` - (the proposed data model acts as a pivot data model) +- simplify the development of a new :ref:`Strategy ` for a + given :ref:`Goal ` when there already are some existing + :ref:`Strategies ` associated to the same :ref:`Goal + ` +- avoid duplicating the same code in several :ref:`Strategies + ` associated to the same :ref:`Goal ` +- have a better consistency between the different :ref:`Strategies + ` for a given :ref:`Goal ` +- avoid any strong coupling with any external :ref:`Cluster Data Model + ` (the proposed data model acts as a pivot + data model) -There may be various -:ref:`generic and basic Cluster Data Models ` -proposed in Watcher helpers, each of them being adapted to achieving a given -:ref:`Goal `: +There may be various :ref:`generic and basic Cluster Data Models +` proposed in Watcher helpers, each of them +being adapted to achieving a given :ref:`Goal `: -- For example, for a - :ref:`Goal ` which aims at optimizing the network - :ref:`resources ` the - :ref:`Strategy ` may need to know which - :ref:`resources ` are communicating together. -- Whereas for a :ref:`Goal ` which aims at optimizing thermal - and power conditions, the :ref:`Strategy ` may need to - know the location of each compute node in the racks and the location of each - rack in the room. +- For example, for a :ref:`Goal ` which aims at optimizing + the network :ref:`resources ` the :ref:`Strategy + ` may need to know which :ref:`resources + ` are communicating together. +- Whereas for a :ref:`Goal ` which aims at optimizing thermal + and power conditions, the :ref:`Strategy ` may need to + know the location of each compute node in the racks and the location of each + rack in the room. -Note however that a developer can use his/her own -:ref:`Cluster Data Model ` if the proposed data -model does not fit his/her needs as long as the -:ref:`Strategy ` is able to produce a -:ref:`Solution ` for the requested -:ref:`Goal `. -For example, a developer could rely on the Nova Data Model to optimize some -compute resources. +Note however that a developer can use his/her own :ref:`Cluster Data Model +` if the proposed data model does not fit +his/her needs as long as the :ref:`Strategy ` is able to +produce a :ref:`Solution ` for the requested :ref:`Goal +`. For example, a developer could rely on the Nova Data Model +to optimize some compute resources. The :ref:`Cluster Data Model ` may be persisted in any appropriate storage system (SQL database, NoSQL database, JSON file, -XML File, In Memory Database, ...). +XML File, In Memory Database, ...). As of now, an in-memory model is built and +maintained in the background in order to accelerate the execution of +strategies. """ import abc diff --git a/watcher/opts.py b/watcher/opts.py index 5233e61fb..397d60867 100644 --- a/watcher/opts.py +++ b/watcher/opts.py @@ -26,6 +26,7 @@ from watcher.common import utils from watcher.decision_engine.loading import default as decision_engine_loader from watcher.decision_engine import manager as decision_engine_manger from watcher.decision_engine.planner import manager as planner_manager +from watcher.metrics_engine.loading import default as cdm_loader PLUGIN_LOADERS = ( @@ -33,6 +34,7 @@ PLUGIN_LOADERS = ( decision_engine_loader.DefaultPlannerLoader, decision_engine_loader.DefaultStrategyLoader, applier_loader.DefaultWorkFlowEngineLoader, + cdm_loader.ClusterDataModelCollectorLoader, )