Supports fetching API endpoints from mDNS

This change enables IPA to receive API endpoints and configuration
via multicast DNS.

Story: #2005393
Task: #30382
Change-Id: Ibbf07052bea8f5c0305dda098b2879bcbc2fece5

ironic_python_agent/agent.py

@@ -21,6 +21,8 @@ import threading
 import time
 from wsgiref import simple_server
 
+from ironic_lib import exception as lib_exc
+from ironic_lib import mdns
 import netaddr
 from oslo_concurrency import processutils
 from oslo_config import cfg
@@ -31,6 +33,7 @@ from six.moves.urllib import parse as urlparse
 from stevedore import extension
 
 from ironic_python_agent.api import app
+from ironic_python_agent import config
 from ironic_python_agent import encoding
 from ironic_python_agent import errors
 from ironic_python_agent.extensions import base
@@ -176,6 +179,21 @@ class IronicPythonAgent(base.ExecuteCommandMixin):
             invoke_kwds={'agent': self},
         )
         self.api_url = api_url
+        if not self.api_url or self.api_url == 'mdns':
+            try:
+                self.api_url, params = mdns.get_endpoint('baremetal')
+            except lib_exc.ServiceLookupFailure:
+                if self.api_url:
+                    # mDNS explicitly requested, report failure.
+                    raise
+                else:
+                    # implicit fallback to mDNS, do not fail (maybe we're only
+                    # running inspection).
+                    LOG.warning('Could not get baremetal endpoint from mDNS, '
+                                'will not heartbeat')
+            else:
+                config.override(params)
+
         if self.api_url:
             self.api_client = ironic_api_client.APIClient(self.api_url)
             self.heartbeater = IronicPythonAgentHeartbeater(self)

ironic_python_agent/config.py

@@ -13,23 +13,29 @@
 # limitations under the License.
 
 from oslo_config import cfg
+from oslo_log import log as logging
 
-from ironic_python_agent import inspector
 from ironic_python_agent import netutils
 from ironic_python_agent import utils
 
+LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 APARAMS = utils.get_agent_params()
 
+INSPECTION_DEFAULT_COLLECTOR = 'default'
+INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT = 60
+
 cli_opts = [
     cfg.StrOpt('api_url',
                default=APARAMS.get('ipa-api-url'),
                deprecated_name='api-url',
-               regex='^http(s?):\\/\\/.+',
+               regex='^(mdns|http(s?):\\/\\/.+)',
                help='URL of the Ironic API. '
                     'Can be supplied as "ipa-api-url" kernel parameter.'
-                    'The value must start with either http:// or https://.'),
+                    'The value must start with either http:// or https://. '
+                    'A special value "mdns" can be specified to fetch the '
+                    'URL using multicast DNS service discovery.'),
 
     cfg.StrOpt('listen_host',
                default=APARAMS.get('ipa-listen-host',
@@ -133,12 +139,14 @@ cli_opts = [
                help='Endpoint of ironic-inspector. If set, hardware inventory '
                     'will be collected and sent to ironic-inspector '
                     'on start up. '
+                    'A special value "mdns" can be specified to fetch the '
+                    'URL using multicast DNS service discovery. '
                     'Can be supplied as "ipa-inspection-callback-url" '
                     'kernel parameter.'),
 
     cfg.StrOpt('inspection_collectors',
                default=APARAMS.get('ipa-inspection-collectors',
-                                   inspector.DEFAULT_COLLECTOR),
+                                   INSPECTION_DEFAULT_COLLECTOR),
                help='Comma-separated list of plugins providing additional '
                     'hardware data for inspection, empty value gives '
                     'a minimum required set of plugins. '
@@ -148,7 +156,7 @@ cli_opts = [
     cfg.IntOpt('inspection_dhcp_wait_timeout',
                min=0,
                default=APARAMS.get('ipa-inspection-dhcp-wait-timeout',
-                                   inspector.DEFAULT_DHCP_WAIT_TIMEOUT),
+                                   INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT),
                help='Maximum time (in seconds) to wait for the PXE NIC '
                     '(or all NICs if inspection_dhcp_all_interfaces is True) '
                     'to get its IP address via DHCP before inspection. '
@@ -216,3 +224,29 @@ CONF.register_cli_opts(cli_opts)
 
 def list_opts():
     return [('DEFAULT', cli_opts)]
+
+
+def override(params):
+    """Override configuration with values from a dictionary.
+
+    This is used for configuration overrides from mDNS.
+
+    :param params: new configuration parameters as a dict.
+    """
+    if not params:
+        return
+
+    LOG.debug('Overriding configuration with %s', params)
+    for key, value in params.items():
+        if key.startswith('ipa_'):
+            key = key[4:]
+        else:
+            LOG.warning('Skipping unknown configuration option %s', key)
+            continue
+
+        try:
+            CONF.set_override(key, value)
+        except Exception as exc:
+            LOG.warning('Unable to override configuration option %(key)s '
+                        'with %(value)r: %(exc)s',
+                        {'key': key, 'value': value, 'exc': exc})

ironic_python_agent/inspector.py

@@ -16,6 +16,7 @@
 import os
 import time
 
+from ironic_lib import mdns
 from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -24,6 +25,7 @@ from oslo_utils import excutils
 import requests
 import stevedore
 
+from ironic_python_agent import config
 from ironic_python_agent import encoding
 from ironic_python_agent import errors
 from ironic_python_agent import hardware
@@ -32,8 +34,6 @@ from ironic_python_agent import utils
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
-DEFAULT_COLLECTOR = 'default'
-DEFAULT_DHCP_WAIT_TIMEOUT = 60
 
 _DHCP_RETRY_INTERVAL = 2
 _COLLECTOR_NS = 'ironic_python_agent.inspector.collectors'
@@ -63,6 +63,15 @@ def inspect():
     if not CONF.inspection_callback_url:
         LOG.info('Inspection is disabled, skipping')
         return
+
+    if CONF.inspection_callback_url == 'mdns':
+        LOG.debug('Fetching the inspection URL from mDNS')
+        url, params = mdns.get_endpoint('baremetal-introspection')
+        # We expect a proper catalog URL, which doesn't include any path.
+        CONF.set_override('inspection_callback_url',
+                          url.rstrip('/') + '/v1/continue')
+        config.override(params)
+
     collector_names = [x.strip() for x in CONF.inspection_collectors.split(',')
                        if x.strip()]
     LOG.info('inspection is enabled with collectors %s', collector_names)

ironic_python_agent/tests/unit/test_agent.py

@@ -16,6 +16,7 @@ import socket
 import time
 from wsgiref import simple_server
 
+from ironic_lib import exception as lib_exc
 import mock
 from oslo_concurrency import processutils
 from oslo_config import cfg
@@ -217,6 +218,126 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest):
                          mock_dispatch.call_args_list)
         self.agent.heartbeater.start.assert_called_once_with()
 
+    @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True)
+    @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock())
+    @mock.patch(
+        'ironic_python_agent.hardware_managers.cna._detect_cna_card',
+        mock.Mock())
+    @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True)
+    @mock.patch.object(agent.IronicPythonAgent,
+                       '_wait_for_interface', autospec=True)
+    @mock.patch('wsgiref.simple_server.WSGIServer', autospec=True)
+    @mock.patch.object(hardware, 'load_managers', autospec=True)
+    def test_url_from_mdns_by_default(self, mock_load_managers, mock_wsgi,
+                                      mock_wait, mock_dispatch, mock_mdns):
+        CONF.set_override('inspection_callback_url', '')
+        mock_mdns.return_value = 'https://example.com', {}
+
+        wsgi_server = mock_wsgi.return_value
+
+        self.agent = agent.IronicPythonAgent(None,
+                                             agent.Host('203.0.113.1', 9990),
+                                             agent.Host('192.0.2.1', 9999),
+                                             3,
+                                             10,
+                                             'eth0',
+                                             300,
+                                             1,
+                                             False)
+
+        def set_serve_api():
+            self.agent.serve_api = False
+
+        wsgi_server.handle_request.side_effect = set_serve_api
+        self.agent.heartbeater = mock.Mock()
+        self.agent.api_client.lookup_node = mock.Mock()
+        self.agent.api_client.lookup_node.return_value = {
+            'node': {
+                'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
+            },
+            'config': {
+                'heartbeat_timeout': 300
+            }
+        }
+
+        self.agent.run()
+
+        listen_addr = agent.Host('192.0.2.1', 9999)
+        mock_wsgi.assert_called_once_with(
+            (listen_addr.hostname,
+             listen_addr.port),
+            simple_server.WSGIRequestHandler)
+        wsgi_server.set_app.assert_called_once_with(self.agent.api)
+        self.assertTrue(wsgi_server.handle_request.called)
+        mock_wait.assert_called_once_with(mock.ANY)
+        self.assertEqual([mock.call('list_hardware_info'),
+                          mock.call('wait_for_disks')],
+                         mock_dispatch.call_args_list)
+        self.agent.heartbeater.start.assert_called_once_with()
+
+    @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True)
+    @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock())
+    @mock.patch(
+        'ironic_python_agent.hardware_managers.cna._detect_cna_card',
+        mock.Mock())
+    @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True)
+    @mock.patch.object(agent.IronicPythonAgent,
+                       '_wait_for_interface', autospec=True)
+    @mock.patch('wsgiref.simple_server.WSGIServer', autospec=True)
+    @mock.patch.object(hardware, 'load_managers', autospec=True)
+    def test_url_from_mdns_explicitly(self, mock_load_managers, mock_wsgi,
+                                      mock_wait, mock_dispatch, mock_mdns):
+        CONF.set_override('inspection_callback_url', '')
+        CONF.set_override('disk_wait_attempts', 0)
+        mock_mdns.return_value = 'https://example.com', {
+            # configuration via mdns
+            'ipa_disk_wait_attempts': '42',
+        }
+
+        wsgi_server = mock_wsgi.return_value
+
+        self.agent = agent.IronicPythonAgent('mdns',
+                                             agent.Host('203.0.113.1', 9990),
+                                             agent.Host('192.0.2.1', 9999),
+                                             3,
+                                             10,
+                                             'eth0',
+                                             300,
+                                             1,
+                                             False)
+
+        def set_serve_api():
+            self.agent.serve_api = False
+
+        wsgi_server.handle_request.side_effect = set_serve_api
+        self.agent.heartbeater = mock.Mock()
+        self.agent.api_client.lookup_node = mock.Mock()
+        self.agent.api_client.lookup_node.return_value = {
+            'node': {
+                'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c'
+            },
+            'config': {
+                'heartbeat_timeout': 300
+            }
+        }
+
+        self.agent.run()
+
+        listen_addr = agent.Host('192.0.2.1', 9999)
+        mock_wsgi.assert_called_once_with(
+            (listen_addr.hostname,
+             listen_addr.port),
+            simple_server.WSGIRequestHandler)
+        wsgi_server.set_app.assert_called_once_with(self.agent.api)
+        self.assertTrue(wsgi_server.handle_request.called)
+        mock_wait.assert_called_once_with(mock.ANY)
+        self.assertEqual([mock.call('list_hardware_info'),
+                          mock.call('wait_for_disks')],
+                         mock_dispatch.call_args_list)
+        self.agent.heartbeater.start.assert_called_once_with()
+        # changed via mdns
+        self.assertEqual(42, CONF.disk_wait_attempts)
+
     @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock())
     @mock.patch(
         'ironic_python_agent.hardware_managers.cna._detect_cna_card',
@@ -314,6 +435,7 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest):
                          mock_dispatch.call_args_list)
         self.agent.heartbeater.start.assert_called_once_with()
 
+    @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True)
     @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock())
     @mock.patch(
         'ironic_python_agent.hardware_managers.cna._detect_cna_card',
@@ -330,7 +452,9 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest):
                                                 mock_wsgi,
                                                 mock_dispatch,
                                                 mock_inspector,
-                                                mock_wait):
+                                                mock_wait,
+                                                mock_mdns):
+        mock_mdns.side_effect = lib_exc.ServiceLookupFailure()
         # If inspection_callback_url is configured and api_url is not when the
         # agent starts, ensure that the inspection will be called and wsgi
         # server will work as usual. Also, make sure api_client and heartbeater
@@ -369,6 +493,7 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest):
         self.assertFalse(mock_wait.called)
         self.assertFalse(mock_dispatch.called)
 
+    @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True)
     @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock())
     @mock.patch(
         'ironic_python_agent.hardware_managers.cna._detect_cna_card',
@@ -385,7 +510,9 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest):
                                                mock_wsgi,
                                                mock_dispatch,
                                                mock_inspector,
-                                               mock_wait):
+                                               mock_wait,
+                                               mock_mdns):
+        mock_mdns.side_effect = lib_exc.ServiceLookupFailure()
         # If both api_url and inspection_callback_url are not configured when
         # the agent starts, ensure that the inspection will be skipped and wsgi
         # server will work as usual. Also, make sure api_client and heartbeater

ironic_python_agent/tests/unit/test_inspector.py

@@ -24,6 +24,7 @@ from oslo_config import cfg
 import requests
 import stevedore
 
+from ironic_python_agent import config
 from ironic_python_agent import errors
 from ironic_python_agent import hardware
 from ironic_python_agent import inspector
@@ -48,8 +49,9 @@ class AcceptingFailure(mock.Mock):
 
 class TestMisc(base.IronicAgentTest):
     def test_default_collector_loadable(self):
-        ext = inspector.extension_manager([inspector.DEFAULT_COLLECTOR])
+        ext = inspector.extension_manager(
-        self.assertIs(ext[inspector.DEFAULT_COLLECTOR].plugin,
+            [config.INSPECTION_DEFAULT_COLLECTOR])
+        self.assertIs(ext[config.INSPECTION_DEFAULT_COLLECTOR].plugin,
                       inspector.collect_default)
 
     def test_raise_on_wrong_collector(self):
@@ -80,6 +82,26 @@ class TestInspect(base.IronicAgentTest):
         mock_call.assert_called_with_failure()
         self.assertEqual('uuid1', result)
 
+    @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True)
+    def test_mdns(self, mock_mdns, mock_ext_mgr, mock_call):
+        CONF.set_override('inspection_callback_url', 'mdns')
+        mock_mdns.return_value = 'http://example', {
+            'ipa_inspection_collectors': 'one,two'
+        }
+        mock_ext_mgr.return_value = [self.mock_ext]
+        mock_call.return_value = {'uuid': 'uuid1'}
+
+        result = inspector.inspect()
+
+        self.mock_collect.assert_called_with_failure()
+        mock_call.assert_called_with_failure()
+        self.assertEqual('uuid1', result)
+
+        self.assertEqual('http://example/v1/continue',
+                         CONF.inspection_callback_url)
+        self.assertEqual('one,two', CONF.inspection_collectors)
+        self.assertEqual(['one', 'two'], mock_ext_mgr.call_args[1]['names'])
+
     def test_collectors_option(self, mock_ext_mgr, mock_call):
         CONF.set_override('inspection_collectors', 'foo,bar')
         mock_ext_mgr.return_value = [
@@ -362,7 +384,7 @@ class TestWaitForDhcp(base.IronicAgentTest):
     def setUp(self):
         super(TestWaitForDhcp, self).setUp()
         CONF.set_override('inspection_dhcp_wait_timeout',
-                          inspector.DEFAULT_DHCP_WAIT_TIMEOUT)
+                          config.INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT)
 
     @mock.patch.object(time, 'sleep', autospec=True)
     def test_all(self, mocked_sleep, mocked_dispatch):

lower-constraints.txt

@@ -24,7 +24,7 @@ greenlet==0.4.13
 hacking==1.0.0
 idna==2.6
 imagesize==1.0.0
-ironic-lib==2.16.0
+ironic-lib==2.17.0
 iso8601==0.1.11
 Jinja2==2.10
 keystoneauth1==3.4.0

releasenotes/notes/mdns-e020484e64d76edb.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Supports fetching baremetal and baremetal introspection endpoints from
+    mDNS instead of providing them via kernel parameters or a configuration
+    file. See `story 2005393
+    <https://storyboard.openstack.org/#!/story/2005393>`_ for more details.
+upgrade:
+  - |
+    When no baremetal API URL is provided (e.g. via the ``ipa-api-url`` kernel
+    parameter), ironic-python-agent now tries to get the URL using mDNS service
+    discovery.

requirements.txt

@@ -21,4 +21,4 @@ rtslib-fb>=2.1.65 # Apache-2.0
 six>=1.10.0 # MIT
 stevedore>=1.20.0 # Apache-2.0
 WSME>=0.8.0 # MIT
-ironic-lib>=2.16.0 # Apache-2.0
+ironic-lib>=2.17.0 # Apache-2.0