Support reapply with supplied introspection data

This patch adds support to provide unprocessed introspection data
to reapply a node. The provided introspection data will be save to
current introspection storage backend.

Change-Id: I969ae9c32f53f89c006a64a006388ddea9542aa5
Story: 1564863
Task: 11344

api-ref/source/introspection-api-v1-introspection-management.inc

@@ -70,28 +70,32 @@ The response will contain introspection data in the form of json string.
    :language: javascript
 
 
-Reapply Introspection on stored data
+Reapply Introspection on data
-====================================
+=============================
 
 .. rest_method::  POST /v1/introspection/{node_id}/data/unprocessed
 
-This method triggers introspection on stored unprocessed data.
+This method triggers introspection on either stored introspection data or raw
-No data is allowed to be sent along with the request.
+introspection data provided in the request. If the introspection data is
+provided in the request body, it should be a valid JSON with content similar to
+ramdisk callback request.
+
+.. versionadded:: 1.15
+  Unprocessed introspection data can be sent via request body.
 
 .. note::
-
+    Reapplying introspection on stored data is only possible when a storage
-    Requires enabling introspection storage backend via ``[processing]store_data``.
+    backend is enabled via ``[processing]store_data``.
 
 Normal response codes: 202
 
 Error codes:
 
-* 400 - bad request or store not configured
+* 400 - bad request, store not configured or malformed data in request body
 * 401, 403 - missing or invalid authentication
 * 404 - node not found for Node ID
 * 409 - inspector locked node for processing
 
-
 Request
 -------
 

doc/source/user/http-api.rst

@@ -398,4 +398,4 @@ Version History
 * **1.13** adds ``manage_boot`` parameter for the introspection API.
 * **1.14** allows formatting to be applied to strings nested in dicts and lists
   in the actions of introspection rules.
-
+* **1.15** allows reapply with provided introspection data from request.

ironic_inspector/common/rpc.py

@@ -33,7 +33,7 @@ def get_transport():
 def get_client():
     """Get a RPC client instance."""
     target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
-                              version='1.1')
+                              version='1.2')
     transport = get_transport()
     return messaging.RPCClient(transport, target)
 
@@ -43,7 +43,7 @@ def get_server(endpoints):
 
     transport = get_transport()
     target = messaging.Target(topic=manager.MANAGER_TOPIC, server=CONF.host,
-                              version='1.1')
+                              version='1.2')
     return messaging.get_rpc_server(
         transport, target, endpoints, executor='eventlet',
         access_policy=dispatcher.DefaultRPCAccessPolicy)

ironic_inspector/conductor/manager.py

@@ -39,7 +39,7 @@ MANAGER_TOPIC = 'ironic-inspector-conductor'
 
 class ConductorManager(object):
     """ironic inspector conductor manager"""
-    RPC_API_VERSION = '1.1'
+    RPC_API_VERSION = '1.2'
 
     target = messaging.Target(version=RPC_API_VERSION)
 
@@ -126,16 +126,21 @@ class ConductorManager(object):
         introspect.abort(node_id, token=token)
 
     @messaging.expected_exceptions(utils.Error)
-    def do_reapply(self, context, node_uuid, token=None):
+    def do_reapply(self, context, node_uuid, token=None, data=None):
+        if not data:
             try:
-            data = process.get_introspection_data(node_uuid, processed=False,
+                data = process.get_introspection_data(node_uuid,
+                                                      processed=False,
                                                       get_json=True)
             except utils.IntrospectionDataStoreDisabled:
                 raise utils.Error(_('Inspector is not configured to store '
-                                'data. Set the [processing]store_data '
+                                    'introspection data. Set the '
-                                'configuration option to change this.'),
+                                    '[processing]store_data configuration '
-                              code=400)
+                                    'option to change this.'))
-        process.reapply(node_uuid, data)
+        else:
+            process.store_introspection_data(node_uuid, data, processed=False)
+
+        process.reapply(node_uuid, data=data)
 
 
 def periodic_clean_up():  # pragma: no cover

ironic_inspector/main.py

@@ -39,7 +39,7 @@ app = flask.Flask(__name__)
 LOG = utils.getProcessingLogger(__name__)
 
 MINIMUM_API_VERSION = (1, 0)
-CURRENT_API_VERSION = (1, 14)
+CURRENT_API_VERSION = (1, 15)
 DEFAULT_API_VERSION = CURRENT_API_VERSION
 _LOGGING_EXCLUDED_KEYS = ('logs',)
 
@@ -307,14 +307,25 @@ def api_introspection_data(node_id):
 @api('/v1/introspection/<node_id>/data/unprocessed',
      rule="introspection:reapply", methods=['POST'])
 def api_introspection_reapply(node_id):
+    data = None
     if flask.request.content_length:
-        return error_response(_('User data processing is not '
+        try:
-                                'supported yet'), code=400)
+            data = flask.request.get_json(force=True)
+        except Exception:
+            raise utils.Error(
+                _('Invalid data: expected a JSON object, got %s') % data)
+        if not isinstance(data, dict):
+            raise utils.Error(
+                _('Invalid data: expected a JSON object, got %s') %
+                data.__class__.__name__)
+        LOG.debug("Received reapply data from request", data=data)
+
     if not uuidutils.is_uuid_like(node_id):
         node = ir_utils.get_node(node_id, fields=['uuid'])
         node_id = node.uuid
+
     client = rpc.get_client()
-    client.call({}, 'do_reapply', node_uuid=node_id)
+    client.call({}, 'do_reapply', node_uuid=node_id, data=data)
     return '', 202
 
 

ironic_inspector/process.py

@@ -140,7 +140,15 @@ def _filter_data_excluded_keys(data):
             if k not in _STORAGE_EXCLUDED_KEYS}
 
 
-def _store_data(node_uuid, data, processed=True):
+def store_introspection_data(node_uuid, data, processed=True):
+    """Store introspection data to the storage backend.
+
+    :param node_uuid: node UUID
+    :param data: Introspection data to be saved
+    :param processed: The type of introspection data, set to True means the
+                      introspection data is processed, otherwise unprocessed.
+    :raises: utils.Error
+    """
     introspection_data_manager = plugins_base.introspection_data_manager()
     store = CONF.processing.store_data
     ext = introspection_data_manager[store].obj
@@ -150,13 +158,22 @@ def _store_data(node_uuid, data, processed=True):
 def _store_unprocessed_data(node_uuid, data):
     # runs in background
     try:
-        _store_data(node_uuid, data, processed=False)
+        store_introspection_data(node_uuid, data, processed=False)
     except Exception:
         LOG.exception('Encountered exception saving unprocessed '
                       'introspection data for node %s', node_uuid, data=data)
 
 
 def get_introspection_data(uuid, processed=True, get_json=False):
+    """Get introspection data from the storage backend.
+
+    :param uuid: node UUID
+    :param processed: Indicates the type of introspection data to be read,
+                      set True to request processed introspection data.
+    :param get_json: Specify whether return the introspection data in json
+                     format, string value is returned if False.
+    :raises: utils.Error
+    """
     introspection_data_manager = plugins_base.introspection_data_manager()
     store = CONF.processing.store_data
     ext = introspection_data_manager[store].obj
@@ -242,7 +259,7 @@ def _process_node(node_info, node, introspection_data):
     # NOTE(dtantsur): repeat the check in case something changed
     ir_utils.check_provision_state(node)
     _run_post_hooks(node_info, introspection_data)
-    _store_data(node_info.uuid, introspection_data)
+    store_introspection_data(node_info.uuid, introspection_data)
 
     ironic = ir_utils.get_client()
     pxe_filter.driver().sync(ironic)
@@ -292,8 +309,8 @@ def reapply(node_uuid, data=None):
     stored data.
 
     :param node_uuid: node UUID
+    :param data: unprocessed introspection data to be reapplied
     :raises: utils.Error
-
     """
 
     LOG.debug('Processing re-apply introspection request for node '
@@ -307,7 +324,7 @@ def reapply(node_uuid, data=None):
         raise utils.Error(_('Node locked, please, try again later'),
                           node_info=node_info, code=409)
 
-    utils.executor().submit(_reapply, node_info, data)
+    utils.executor().submit(_reapply, node_info, introspection_data=data)
 
 
 def _reapply(node_info, introspection_data=None):
@@ -350,6 +367,6 @@ def _reapply_with_data(node_info, introspection_data):
                           '\n'.join(failures), node_info=node_info)
 
     _run_post_hooks(node_info, introspection_data)
-    _store_data(node_info.uuid, introspection_data)
+    store_introspection_data(node_info.uuid, introspection_data)
     node_info.invalidate_cache()
     rules.apply(node_info, introspection_data)

ironic_inspector/test/functional.py

@@ -634,6 +634,14 @@ class Test(Base):
         self.assertEqual(store_processing_call,
                          store_mock.call_args_list[-1])
 
+        # Reapply with provided data
+        res = self.call_reapply(self.uuid, data=copy.deepcopy(self.data))
+        self.assertEqual(202, res.status_code)
+        self.assertEqual('', res.text)
+        eventlet.greenthread.sleep(DEFAULT_SLEEP)
+
+        self.check_status(status, finished=True, state=istate.States.finished)
+
     @mock.patch.object(swift, 'store_introspection_data', autospec=True)
     @mock.patch.object(swift, 'get_introspection_data', autospec=True)
     def test_edge_state_transitions(self, get_mock, store_mock):

ironic_inspector/test/unit/test_main.py

@@ -370,17 +370,26 @@ class TestApiReapply(BaseAPITest):
         self.app.post('/v1/introspection/%s/data/unprocessed' %
                       self.uuid)
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
-                                                      node_uuid=self.uuid)
+                                                      node_uuid=self.uuid,
+                                                      data=None)
 
     def test_user_data(self):
         res = self.app.post('/v1/introspection/%s/data/unprocessed' %
                             self.uuid, data='some data')
         self.assertEqual(400, res.status_code)
         message = json.loads(res.data.decode())['error']['message']
-        self.assertEqual('User data processing is not supported yet',
+        self.assertIn('Invalid data: expected a JSON object', message)
-                         message)
         self.assertFalse(self.client_mock.call.called)
 
+    def test_user_data_valid(self):
+        data = {"foo": "bar"}
+        res = self.app.post('/v1/introspection/%s/data/unprocessed' %
+                            self.uuid, data=json.dumps(data))
+        self.assertEqual(202, res.status_code)
+        self.client_mock.call.assert_called_once_with({}, 'do_reapply',
+                                                      node_uuid=self.uuid,
+                                                      data=data)
+
     def test_get_introspection_data_error(self):
         exc = utils.Error('The store is crashed', code=404)
         self.client_mock.call.side_effect = exc
@@ -392,7 +401,8 @@ class TestApiReapply(BaseAPITest):
         message = json.loads(res.data.decode())['error']['message']
         self.assertEqual(str(exc), message)
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
-                                                      node_uuid=self.uuid)
+                                                      node_uuid=self.uuid,
+                                                      data=None)
 
     def test_generic_error(self):
         exc = utils.Error('Oops', code=400)
@@ -405,7 +415,8 @@ class TestApiReapply(BaseAPITest):
         message = json.loads(res.data.decode())['error']['message']
         self.assertEqual(str(exc), message)
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
-                                                      node_uuid=self.uuid)
+                                                      node_uuid=self.uuid,
+                                                      data=None)
 
     @mock.patch.object(ir_utils, 'get_node', autospec=True)
     def test_reapply_with_node_name(self, get_mock):
@@ -413,7 +424,8 @@ class TestApiReapply(BaseAPITest):
         self.app.post('/v1/introspection/%s/data/unprocessed' %
                       'fake-node')
         self.client_mock.call.assert_called_once_with({}, 'do_reapply',
-                                                      node_uuid=self.uuid)
+                                                      node_uuid=self.uuid,
+                                                      data=None)
         get_mock.assert_called_once_with('fake-node', fields=['uuid'])
 
 

ironic_inspector/test/unit/test_manager.py

@@ -386,8 +386,8 @@ class TestManagerReapply(BaseManagerTest):
                                 self.context, self.uuid)
 
         self.assertEqual(utils.Error, exc.exc_info[0])
-        self.assertIn('Inspector is not configured to store data',
+        self.assertIn('Inspector is not configured to store introspection '
-                      str(exc.exc_info[1]))
+                      'data', str(exc.exc_info[1]))
         self.assertEqual(400, exc.exc_info[1].http_code)
         self.assertFalse(reapply_mock.called)
 
@@ -407,3 +407,12 @@ class TestManagerReapply(BaseManagerTest):
         reapply_mock.assert_called_once_with(self.uuid, data=self.data)
         get_data_mock.assert_called_once_with(self.uuid, processed=False,
                                               get_json=True)
+
+    @mock.patch.object(process, 'store_introspection_data', autospec=True)
+    @mock.patch.object(process, 'get_introspection_data', autospec=True)
+    def test_reapply_with_data(self, get_mock, store_mock, reapply_mock):
+        self.manager.do_reapply(self.context, self.uuid, data=self.data)
+        reapply_mock.assert_called_once_with(self.uuid, data=self.data)
+        store_mock.assert_called_once_with(self.uuid, self.data,
+                                           processed=False)
+        self.assertFalse(get_mock.called)

ironic_inspector/test/unit/test_process.py

@@ -584,6 +584,16 @@ class TestReapply(BaseTest):
             blocking=False
         )
 
+    @prepare_mocks
+    def test_reapply_with_data(self, pop_mock, reapply_mock):
+        process.reapply(self.uuid, data=self.data)
+        pop_mock.assert_called_once_with(self.uuid, locked=False)
+        pop_mock.return_value.acquire_lock.assert_called_once_with(
+            blocking=False
+        )
+        reapply_mock.assert_called_once_with(pop_mock.return_value,
+                                             introspection_data=self.data)
+
 
 @mock.patch.object(example_plugin.ExampleProcessingHook, 'before_update')
 @mock.patch.object(process.rules, 'apply', autospec=True)

releasenotes/notes/post-introspection-data-9cdd39a3de446e92.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds support to reapply with provided unprocessed introspection data. The
+    introspection data is supplied in the body of POST request to
+    ``/v1/introspection/<node_id>/data/unprocessed``. The introspection data
+    will also be saved to storage backend.