Update oslo_config integration to support auto-generation of files

drydock_provisioner/__init__.py

@@ -11,8 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from .config import DrydockConfig
-
-config_mgr = DrydockConfig()
-conf = config_mgr.conf

drydock_provisioner/config.py

@@ -12,19 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+"""Single point of entry to generate the sample configuration file.
+
+This module collects all the necessary info from the other modules in this
+package. It is assumed that:
+
+* Every other module in this package has a 'list_opts' function which
+  returns a dict where:
+
+  * The keys are strings which are the group names.
+
+  * The value of each key is a list of config options for that group.
+
+* The conf package doesn't have further packages with config options.
+
+* This module is only used in the context of sample file generation.
+
+"""
+import collections
+import importlib
+import os
+import pkgutil
+
 from oslo_config import cfg
 
 class DrydockConfig(object):
     """
     Initialize all the core options
     """
-    # Core/General options
+    # Logging options
     logging_options = [
         cfg.StrOpt('log_level', default='INFO', help='Global log level for Drydock'),
         cfg.StrOpt('global_logger_name', default='drydock', help='Logger name for the top-level logger'),
-        cfg.StrOpt('oobdriver_logger_name', default='${global_logger_name}.oobdriver'),
-        cfg.StrOpt('nodedriver_logger_name', default='${global_logger_name}.nodedriver'),
-        cfg.StrOpt('control_logger_name', default='${global_logger_name}.control'),
+        cfg.StrOpt('oobdriver_logger_name', default='${global_logger_name}.oobdriver', help='Logger name for OOB driver logging'),
+        cfg.StrOpt('nodedriver_logger_name', default='${global_logger_name}.nodedriver', help='Logger name for Node driver logging'),
+        cfg.StrOpt('control_logger_name', default='${global_logger_name}.control', help='Logger name for API server logging'),
     ]
 
     # API Authentication options
@@ -50,17 +73,72 @@ class DrydockConfig(object):
 
     # Timeouts for various tasks specified in minutes
     timeout_options = [
-        cfg.IntOpt('create_network_template',default=2,help='Timeout in minutes for creating site network templates'),
-        cfg.IntOpt('identify_node',default=10,help='Timeout in minutes for initial node identification'),
-        cfg.IntOpt('configure_hardware',default=30,help='Timeout in minutes for node commissioning and hardware configuration'),
-        cfg.IntOpt('apply_node_networking',default=5,help='Timeout in minutes for configuring node networking'),
-        cfg.IntOpt('deploy_node',default=45,help='Timeout in minutes for deploying a node'),
+        cfg.IntOpt('drydock_timeout', default=5, help='Fallback timeout when a specific one is not configured'),
+        cfg.IntOpt('create_network_template', default=2, help='Timeout in minutes for creating site network templates'),
+        cfg.IntOpt('identify_node', default=10, help='Timeout in minutes for initial node identification'),
+        cfg.IntOpt('configure_hardware', default=30, help='Timeout in minutes for node commissioning and hardware configuration'),
+        cfg.IntOpt('apply_node_networking', default=5, help='Timeout in minutes for configuring node networking'),
+        cfg.IntOpt('deploy_node', default=45, help='Timeout in minutes for deploying a node'),
     ]
 
     def __init__(self):
         self.conf = cfg.ConfigOpts()
 
+    def register_options(self):
         self.conf.register_opts(DrydockConfig.logging_options, group='logging')
         self.conf.register_opts(DrydockConfig.auth_options, group='authentication')
         self.conf.register_opts(DrydockConfig.plugin_options, group='plugins')
         self.conf.register_opts(DrydockConfig.timeout_options, group='timeouts')
+
+config_mgr = DrydockConfig()
+conf = config_mgr.conf
+
+IGNORED_MODULES = ('drydock', 'config')
+
+def list_opts():
+    opts = {'logging': DrydockConfig.logging_options,
+            'authentication': DrydockConfig.auth_options,
+            'plugins': DrydockConfig.plugin_options,
+            'timeouts': DrydockConfig.timeout_options
+        }
+
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    parent_module = ".".join(__name__.split('.')[:-1])
+    module_names = _list_module_names(package_path, parent_module)
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+def _tupleize(d):
+    """Convert a dict of options to the 2-tuple format."""
+    return [(key, value) for key, value in d.items()]
+
+def _list_module_names(pkg_path, parent_module):
+    module_names = []
+    for _, module_name, ispkg in pkgutil.iter_modules(path=[pkg_path]):
+        if module_name in IGNORED_MODULES:
+            # Skip this module.
+            continue
+        elif ispkg:
+            module_names.extend(_list_module_names(pkg_path + "/" + module_name, parent_module + "." + module_name))
+        else:
+            module_names.append(parent_module + "." + module_name)
+    return module_names
+
+def _import_modules(module_names):
+    imported_modules = []
+    for module_name in module_names:
+        module = importlib.import_module(module_name)
+        if hasattr(module, 'list_opts'):
+            print("Pulling options from module %s" % module.__name__)
+            imported_modules.append(module)
+    return imported_modules
+
+def _append_config_options(imported_modules, config_options):
+    for module in imported_modules:
+        configs = module.list_opts()
+        for key, val in configs.items():
+            if key not in config_options:
+                config_options[key] = val
+            else:
+                config_options[key].extend(val)

drydock_provisioner/drivers/__init__.py

@@ -25,6 +25,10 @@ import drydock_provisioner.error as errors
 # driver tasks and feed them via queue
 class ProviderDriver(object):
 
+    driver_name = "generic"
+    driver_key = "generic"
+    driver_desc = "Generic Provider Driver"
+
     def __init__(self, orchestrator=None, state_manager=None, **kwargs):
         if orchestrator is None:
             raise ValueError("ProviderDriver requires valid orchestrator")
@@ -39,9 +43,7 @@ class ProviderDriver(object):
         # These are the actions that this driver supports
         self.supported_actions = [hd_fields.OrchestratorAction.Noop]
 
-        self.driver_name = "generic"
-        self.driver_key = "generic"
-        self.driver_desc = "Generic Provider Driver"
+
 
     def execute_task(self, task_id):
         task = self.state_manager.get_task(task_id)

drydock_provisioner/drivers/node/__init__.py

@@ -20,6 +20,10 @@ from drydock_provisioner.drivers import ProviderDriver
 
 class NodeDriver(ProviderDriver):
 
+    driver_name = "node_generic"
+    driver_key = "node_generic"
+    driver_desc = "Generic Node Driver"
+
     def __init__(self, **kwargs):
         super(NodeDriver, self).__init__(**kwargs)
 
@@ -37,10 +41,6 @@ class NodeDriver(ProviderDriver):
                                   hd_fields.OrchestratorAction.DeployNode,
                                   hd_fields.OrchestratorAction.DestroyNode]
 
-        self.driver_name = "node_generic"
-        self.driver_key = "node_generic"
-        self.driver_desc = "Generic Node Driver"
-
     def execute_task(self, task_id):
         task = self.state_manager.get_task(task_id)
         task_action = task.action

drydock_provisioner/drivers/node/maasdriver/driver.py

@@ -18,7 +18,7 @@ import sys
 
 from oslo_config import cfg
 
-import drydock_provisioner
+import drydock_provisioner.config as config
 import drydock_provisioner.error as errors
 import drydock_provisioner.drivers as drivers
 import drydock_provisioner.objects.fields as hd_fields
@@ -33,26 +33,22 @@ import drydock_provisioner.drivers.node.maasdriver.models.subnet as maas_subnet
 import drydock_provisioner.drivers.node.maasdriver.models.machine as maas_machine
 
 class MaasNodeDriver(NodeDriver):
-
     maasdriver_options = [
         cfg.StrOpt('maas_api_key', help='The API key for accessing MaaS', secret=True),
         cfg.StrOpt('maas_api_url', help='The URL for accessing MaaS API'),
     ]
 
+    driver_name = 'maasdriver'
+    driver_key = 'maasdriver'
+    driver_desc = 'MaaS Node Provisioning Driver'
+
     def __init__(self, **kwargs):
         super(MaasNodeDriver, self).__init__(**kwargs)
 	
-        self.driver_name = "maasdriver"
-        self.driver_key = "maasdriver"
-        self.driver_desc = "MaaS Node Provisioning Driver"
-
-        self.setup_config_options(drydock_provisioner.conf)
+        config.conf.register_opts(maasdriver_options, group='maasdriver')
 
         self.logger = logging.getLogger("%s.%s" %
-                                (drydock_provisioner.conf.logging.nodedriver_logger_name, self.driver_key))
-
-    def setup_config_options(self, conf):
-        conf.register_opts(MaasNodeDriver.maasdriver_options, group=self.driver_key)
+                                (config.conf.logging.nodedriver_logger_name, self.driver_key))
 
     def execute_task(self, task_id):
         task = self.state_manager.get_task(task_id)
@@ -67,7 +63,7 @@ class MaasNodeDriver(NodeDriver):
         if task.action == hd_fields.OrchestratorAction.ValidateNodeServices:
             self.orchestrator.task_field_update(task.get_id(),
                                 status=hd_fields.TaskStatus.Running)
-            maas_client = MaasRequestFactory(drydock_provisioner.conf.maasdriver.maas_api_url, drydock_provisioner.conf.maasdriver.maas_api_key) 
+            maas_client = MaasRequestFactory(config.conf.maasdriver.maas_api_url, config.conf.maasdriver.maas_api_key) 
 
             try:
                 if maas_client.test_connectivity():
@@ -139,7 +135,7 @@ class MaasNodeDriver(NodeDriver):
 
             runner.start()
 
-            runner.join(timeout=drydock_provisioner.conf.timeouts.create_network_template * 60)
+            runner.join(timeout=config.conf.timeouts.create_network_template * 60)
 
             if runner.is_alive():
                 result =  {
@@ -193,7 +189,7 @@ class MaasNodeDriver(NodeDriver):
             attempts = 0
             worked = failed = False
 
-            while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.identify_node:
+            while running_subtasks > 0 and attempts < config.conf.timeouts.identify_node:
                 for t in subtasks:
                     subtask = self.state_manager.get_task(t)
 
@@ -263,7 +259,7 @@ class MaasNodeDriver(NodeDriver):
             worked = failed = False
 
             #TODO Add timeout to config
-            while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.configure_hardware:
+            while running_subtasks > 0 and attempts < config.conf.timeouts.configure_hardware:
                 for t in subtasks:
                     subtask = self.state_manager.get_task(t)
 
@@ -332,7 +328,7 @@ class MaasNodeDriver(NodeDriver):
             attempts = 0
             worked = failed = False
 
-            while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.apply_node_networking:
+            while running_subtasks > 0 and attempts < config.conf.timeouts.apply_node_networking:
                 for t in subtasks:
                     subtask = self.state_manager.get_task(t)
 
@@ -401,7 +397,7 @@ class MaasNodeDriver(NodeDriver):
             attempts = 0
             worked = failed = False
 
-            while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.deploy_node:
+            while running_subtasks > 0 and attempts < config.conf.timeouts.deploy_node:
                 for t in subtasks:
                     subtask = self.state_manager.get_task(t)
 
@@ -453,8 +449,8 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
                             status=hd_fields.TaskStatus.Running,
                             result=hd_fields.ActionResult.Incomplete)
 
-        self.maas_client = MaasRequestFactory(drydock_provisioner.conf.maasdriver.maas_api_url,
-                                              drydock_provisioner.conf.maasdriver.maas_api_key)
+        self.maas_client = MaasRequestFactory(config.conf.maasdriver.maas_api_url,
+                                              config.conf.maasdriver.maas_api_key)
 
         site_design = self.orchestrator.get_effective_site(self.task.design_id)
 
@@ -743,7 +739,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
                             # Poll machine status
                             attempts = 0
 
-                            while attempts < drydock_provisioner.conf.timeouts.configure_hardware and machine.status_name != 'Ready':
+                            while attempts < config.conf.timeouts.configure_hardware and machine.status_name != 'Ready':
                                 attempts = attempts + 1
                                 time.sleep(1 * 60)
                                 try:
@@ -975,7 +971,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
                     continue
 
                 attempts = 0
-                while attempts < drydock_provisioner.conf.timeouts.deploy_node and not machine.status_name.startswith('Deployed'):
+                while attempts < config.conf.timeouts.deploy_node and not machine.status_name.startswith('Deployed'):
                     attempts = attempts + 1
                     time.sleep(1 * 60)
                     try:
@@ -1004,3 +1000,6 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
                                 status=hd_fields.TaskStatus.Complete,
                                 result=final_result,
                                 result_detail=result_detail)
+
+def list_opts():
+    return {MaasNodeDriver.driver_key: MaasNodeDriver.maasdriver_options}

drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py

@@ -100,7 +100,8 @@ class PyghmiDriver(oob.OobDriver):
             runner.start()
 
         attempts = 0
-        while len(incomplete_subtasks) > 0 and attempts <= getattr(drydock_provisioner.conf.timeouts, task.action, 5):
+        while (len(incomplete_subtasks) > 0 and
+              attempts <= getattr(drydock_provisioner.conf.timeouts, task.action, drydock_provisioner.conf.timeouts.drydock_timeout)):
             for n in incomplete_subtasks:
                 t = self.state_manager.get_task(n)
                 if t.get_status() in [hd_fields.TaskStatus.Terminated,

drydock_provisioner/drydock.py

@@ -15,7 +15,7 @@ import logging
 from oslo_config import cfg
 import sys
 
-import drydock_provisioner
+import drydock_provisioner.config as config
 import drydock_provisioner.objects as objects
 import drydock_provisioner.ingester as ingester
 import drydock_provisioner.statemgmt as statemgmt
@@ -30,23 +30,24 @@ def start_drydock():
         cfg.BoolOpt('debug', short='d', default=False, help='Enable debug logging'),
     ]
 
-    drydock_provisioner.conf.register_cli_opts(cli_options)
-    drydock_provisioner.conf(sys.argv[1:])
+    config.conf.register_cli_opts(cli_options)
+    config.config_mgr.register_options()
+    config.conf(sys.argv[1:])
 
-    if drydock_provisioner.conf.debug:
-        drydock_provisioner.conf.logging.log_level = 'DEBUG'
+    if config.conf.debug:
+        config.conf.logging.log_level = 'DEBUG'
 
     # Setup root logger
-    logger = logging.getLogger(drydock_provisioner.conf.logging.global_logger_name)
+    logger = logging.getLogger(config.conf.logging.global_logger_name)
 
-    logger.setLevel(drydock_provisioner.conf.logging.log_level)
+    logger.setLevel(config.conf.logging.log_level)
     ch = logging.StreamHandler()
     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
     ch.setFormatter(formatter)
     logger.addHandler(ch)
 
     # Specalized format for API logging
-    logger = logging.getLogger(drydock_provisioner.conf.logging.control_logger_name)
+    logger = logging.getLogger(config.conf.logging.control_logger_name)
     logger.propagate = False
     formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s')
 
@@ -57,13 +58,13 @@ def start_drydock():
 
     state = statemgmt.DesignState()
 
-    orchestrator = orch.Orchestrator(drydock_provisioner.conf.plugins,
+    orchestrator = orch.Orchestrator(config.conf.plugins,
                              state_manager=state)
     input_ingester = ingester.Ingester()
-    input_ingester.enable_plugins(drydock_provisioner.conf.plugins.ingester)
+    input_ingester.enable_plugins(config.conf.plugins.ingester)
 
     # Now that loggers are configured, log the effective config
-    drydock_provisioner.conf.log_opt_values(logging.getLogger(drydock_provisioner.conf.logging.global_logger_name), logging.DEBUG)
+    config.conf.log_opt_values(logging.getLogger(config.conf.logging.global_logger_name), logging.DEBUG)
 
     return api.start_api(state_manager=state, ingester=input_ingester,
                          orchestrator=orchestrator)

setup.py

@@ -15,20 +15,7 @@
 # drydock_provisioner - A tool to consume a host topology and orchestrate
 # and monitor the provisioning of those hosts and execution of bootstrap
 # scripts
-#
-# Modular services:
-# smelter - 	A service to consume the host topology, will support multiple
-#	 			input formats. Initially supports a YAML schema as demonstrated
-#				in the examples folder
-# tarot -		A service for persisting the host topology and orchestration state
-#				and making the data available via API
-# cockpit -		The entrypoint API for users to control helm-drydock and query
-#				current state
-# alchemist -	The core orchestrator
-# drivers - 	A tree with all of the plugins that alchemist uses to execute
-#				orchestrated tasks
-# jabberwocky -	An introspection API that newly provisioned nodes can use to
-#				ingest self-data and bootstrap their application deployment process
+
 
 from setuptools import setup
 
@@ -65,6 +52,9 @@ setup(name='drydock_provisioner',
         'uwsgi>1.4',
         'bson===0.4.7',
         'oslo.config',
-      ]
+      ],
+      entry_points={
+        'oslo.config.opts': 'drydock_provisioner = drydock_provisioner.config:list_opts',
+      }
      )