Better cleanup for Tempest tests

When running our Tempest tests, we are now cleaning up all the
objects we have created within the WatcherDB via a soft_delete.

Partially Implements: blueprint deletion-of-actions-plan

Change-Id: Ibdcfd2be37094377d09ad77d5c20298ee2baa4d0

watcher_tempest_plugin/services/infra_optim/v1/json/client.py

@@ -174,11 +174,6 @@ class InfraOptimClientJSON(base.BaseInfraOptimClient):
         """Lists details of all existing action plan"""
         return self._list_request('/action_plans/detail', **kwargs)
 
-    @base.handle_errors
-    def list_action_plan_by_audit(self, audit_uuid):
-        """Lists all action plans associated with an audit"""
-        return self._list_request('/action_plans', audit_uuid=audit_uuid)
-
     @base.handle_errors
     def show_action_plan(self, action_plan_uuid):
         """Gets a specific action plan
@@ -190,7 +185,7 @@ class InfraOptimClientJSON(base.BaseInfraOptimClient):
 
     @base.handle_errors
     def delete_action_plan(self, action_plan_uuid):
-        """Deletes an action_plan having the specified UUID
+        """Deletes an action plan having the specified UUID
 
         :param action_plan_uuid: The unique identifier of the action_plan
         :return: A tuple with the server response and the response body
@@ -198,6 +193,18 @@ class InfraOptimClientJSON(base.BaseInfraOptimClient):
 
         return self._delete_request('/action_plans', action_plan_uuid)
 
+    @base.handle_errors
+    def delete_action_plans_by_audit(self, audit_uuid):
+        """Deletes an action plan having the specified UUID
+
+        :param audit_uuid: The unique identifier of the related Audit
+        """
+
+        _, action_plans = self.list_action_plans(audit_uuid=audit_uuid)
+
+        for action_plan in action_plans:
+            self.delete_action_plan(action_plan['uuid'])
+
     @base.handle_errors
     def update_action_plan(self, action_plan_uuid, patch):
         """Update the specified action plan

watcher_tempest_plugin/tests/api/admin/base.py

@@ -18,31 +18,17 @@ import functools
 
 from tempest import test
 from tempest_lib.common.utils import data_utils
-from tempest_lib import exceptions as lib_exc
 
 from watcher_tempest_plugin import infra_optim_clients as clients
 
 
-def creates(resource):
-    """Decorator that adds resources to the appropriate cleanup list."""
-
-    def decorator(f):
-        @functools.wraps(f)
-        def wrapper(cls, *args, **kwargs):
-            resp, body = f(cls, *args, **kwargs)
-
-            if 'uuid' in body:
-                cls.created_objects[resource].add(body['uuid'])
-
-            return resp, body
-        return wrapper
-    return decorator
-
-
 class BaseInfraOptimTest(test.BaseTestCase):
     """Base class for Infrastructure Optimization API tests."""
 
-    RESOURCE_TYPES = ['audit_template', 'audit']
+    # States where the object is waiting for some event to perform a transition
+    IDLE_STATES = ('RECOMMENDED', 'FAILED', 'SUCCEEDED', 'CANCELLED')
+    # States where the object can only be DELETED (end of its life-cycle)
+    FINISHED_STATES = ('FAILED', 'SUCCEEDED', 'CANCELLED')
 
     @classmethod
     def setup_credentials(cls):
@@ -58,19 +44,52 @@ class BaseInfraOptimTest(test.BaseTestCase):
     def resource_setup(cls):
         super(BaseInfraOptimTest, cls).resource_setup()
 
-        cls.created_objects = {}
+        # Set of all created audit templates UUIDs
-        for resource in cls.RESOURCE_TYPES:
+        cls.created_audit_templates = set()
-            cls.created_objects[resource] = set()
+        # Set of all created audit UUIDs
+        cls.created_audits = set()
+        # Set of all created audit UUIDs. We use it to build the list of
+        # action plans to delete (including potential orphan one(s))
+        cls.created_action_plans_audit_uuids = set()
 
     @classmethod
     def resource_cleanup(cls):
         """Ensure that all created objects get destroyed."""
         try:
-            for resource in cls.RESOURCE_TYPES:
+            action_plans_to_be_deleted = set()
-                obj_uuids = cls.created_objects[resource]
+            # Phase 1: Make sure all objects are in an idle state
-                delete_method = getattr(cls.client, 'delete_%s' % resource)
+            for audit_uuid in cls.created_audits:
-                for obj_uuid in obj_uuids:
+                test.call_until_true(
-                    delete_method(obj_uuid, ignore_errors=lib_exc.NotFound)
+                    func=functools.partial(
+                        cls.is_audit_idle, audit_uuid),
+                    duration=30,
+                    sleep_for=.5
+                )
+
+            for audit_uuid in cls.created_action_plans_audit_uuids:
+                _, action_plans = cls.client.list_action_plans(
+                    audit_uuid=audit_uuid)
+                action_plans_to_be_deleted.update(
+                    ap['uuid'] for ap in action_plans['action_plans'])
+
+                for action_plan in action_plans['action_plans']:
+                    test.call_until_true(
+                        func=functools.partial(
+                            cls.is_action_plan_idle, action_plan['uuid']),
+                        duration=30,
+                        sleep_for=.5
+                    )
+
+            # Phase 2: Delete them all
+            for action_plan_uuid in action_plans_to_be_deleted:
+                cls.delete_action_plan(action_plan_uuid)
+
+            for audit_uuid in cls.created_audits.copy():
+                cls.delete_audit(audit_uuid)
+
+            for audit_template_uuid in cls.created_audit_templates.copy():
+                cls.delete_audit_template(audit_template_uuid)
+
         finally:
             super(BaseInfraOptimTest, cls).resource_cleanup()
 
@@ -95,7 +114,6 @@ class BaseInfraOptimTest(test.BaseTestCase):
     # ### AUDIT TEMPLATES ### #
 
     @classmethod
-    @creates('audit_template')
     def create_audit_template(cls, name=None, description=None, goal=None,
                               host_aggregate=None, extra=None):
         """Wrapper utility for creating a test audit template
@@ -111,12 +129,14 @@ class BaseInfraOptimTest(test.BaseTestCase):
             Default: {}
         :return: A tuple with The HTTP response and its body
         """
-
         description = description or data_utils.rand_name(
             'test-audit_template')
         resp, body = cls.client.create_audit_template(
             name=name, description=description, goal=goal,
             host_aggregate=host_aggregate, extra=extra)
+
+        cls.created_audit_templates.add(body['uuid'])
+
         return resp, body
 
     @classmethod
@@ -126,18 +146,16 @@ class BaseInfraOptimTest(test.BaseTestCase):
         :param uuid: The unique identifier of the audit template
         :return: Server response
         """
+        resp, _ = cls.client.delete_audit_template(uuid)
 
-        resp, body = cls.client.delete_audit_template(uuid)
+        if uuid in cls.created_audit_templates:
-
+            cls.created_audit_templates.remove(uuid)
-        if uuid in cls.created_objects['audit_template']:
-            cls.created_objects['audit_template'].remove(uuid)
 
         return resp
 
     # ### AUDITS ### #
 
     @classmethod
-    @creates('audit')
     def create_audit(cls, audit_template_uuid, type='ONESHOT',
                      state='PENDING', deadline=None):
         """Wrapper utility for creating a test audit
@@ -151,6 +169,10 @@ class BaseInfraOptimTest(test.BaseTestCase):
         resp, body = cls.client.create_audit(
             audit_template_uuid=audit_template_uuid, type=type,
             state=state, deadline=deadline)
+
+        cls.created_audits.add(body['uuid'])
+        cls.created_action_plans_audit_uuids.add(body['uuid'])
+
         return resp, body
 
     @classmethod
@@ -160,10 +182,10 @@ class BaseInfraOptimTest(test.BaseTestCase):
         :param audit_uuid: The unique identifier of the audit.
         :return: the HTTP response
         """
-        resp, body = cls.client.delete_audit(audit_uuid)
+        resp, _ = cls.client.delete_audit(audit_uuid)
 
-        if audit_uuid in cls.created_objects['audit']:
+        if audit_uuid in cls.created_audits:
-            cls.created_objects['audit'].remove(audit_uuid)
+            cls.created_audits.remove(audit_uuid)
 
         return resp
 
@@ -172,8 +194,39 @@ class BaseInfraOptimTest(test.BaseTestCase):
         _, audit = cls.client.show_audit(audit_uuid)
         return audit.get('state') == 'SUCCEEDED'
 
+    @classmethod
+    def has_audit_finished(cls, audit_uuid):
+        _, audit = cls.client.show_audit(audit_uuid)
+        return audit.get('state') in cls.FINISHED_STATES
+
+    @classmethod
+    def is_audit_idle(cls, audit_uuid):
+        _, audit = cls.client.show_audit(audit_uuid)
+        return audit.get('state') in cls.IDLE_STATES
+
     # ### ACTION PLANS ### #
 
+    @classmethod
+    def create_action_plan(cls, audit_template_uuid, **audit_kwargs):
+        """Wrapper utility for creating a test action plan
+
+        :param audit_template_uuid: Audit template UUID to use
+        :param audit_kwargs: Dict of audit properties to set
+        :return: The action plan as dict
+        """
+        _, audit = cls.create_audit(audit_template_uuid, **audit_kwargs)
+        audit_uuid = audit['uuid']
+
+        assert test.call_until_true(
+            func=functools.partial(cls.has_audit_succeeded, audit_uuid),
+            duration=30,
+            sleep_for=.5
+        )
+
+        _, action_plans = cls.client.list_action_plans(audit_uuid=audit_uuid)
+
+        return action_plans['action_plans'][0]
+
     @classmethod
     def delete_action_plan(cls, action_plan_uuid):
         """Deletes an action plan having the specified UUID
@@ -181,14 +234,20 @@ class BaseInfraOptimTest(test.BaseTestCase):
         :param action_plan_uuid: The unique identifier of the action plan.
         :return: the HTTP response
         """
-        resp, body = cls.client.delete_action_plan(action_plan_uuid)
+        resp, _ = cls.client.delete_action_plan(action_plan_uuid)
 
-        if action_plan_uuid in cls.created_objects['action_plan']:
+        if action_plan_uuid in cls.created_action_plans_audit_uuids:
-            cls.created_objects['action_plan'].remove(action_plan_uuid)
+            cls.created_action_plans_audit_uuids.remove(action_plan_uuid)
 
         return resp
 
     @classmethod
     def has_action_plan_finished(cls, action_plan_uuid):
         _, action_plan = cls.client.show_action_plan(action_plan_uuid)
-        return action_plan.get('state') in ('FAILED', 'SUCCEEDED', 'CANCELLED')
+        return action_plan.get('state') in cls.FINISHED_STATES
+
+    @classmethod
+    def is_action_plan_idle(cls, action_plan_uuid):
+        """This guard makes sure your action plan is not running"""
+        _, action_plan = cls.client.show_action_plan(action_plan_uuid)
+        return action_plan.get('state') in cls.IDLE_STATES

watcher_tempest_plugin/tests/api/admin/test_action.py

@@ -38,8 +38,8 @@ class TestShowListAction(base.BaseInfraOptimTest):
             duration=30,
             sleep_for=.5
         )
-        _, action_plans = cls.client.list_action_plan_by_audit(
+        _, action_plans = cls.client.list_action_plans(
-            cls.audit['uuid'])
+            audit_uuid=cls.audit['uuid'])
         cls.action_plan = action_plans['action_plans'][0]
 
     @test.attr(type='smoke')

watcher_tempest_plugin/tests/api/admin/test_action_plan.py

@@ -37,7 +37,8 @@ class TestCreateDeleteExecuteActionPlan(base.BaseInfraOptimTest):
             duration=30,
             sleep_for=.5
         ))
-        _, action_plans = self.client.list_action_plan_by_audit(audit['uuid'])
+        _, action_plans = self.client.list_action_plans(
+            audit_uuid=audit['uuid'])
         action_plan = action_plans['action_plans'][0]
 
         _, action_plan = self.client.show_action_plan(action_plan['uuid'])
@@ -55,7 +56,8 @@ class TestCreateDeleteExecuteActionPlan(base.BaseInfraOptimTest):
             duration=30,
             sleep_for=.5
         ))
-        _, action_plans = self.client.list_action_plan_by_audit(audit['uuid'])
+        _, action_plans = self.client.list_action_plans(
+            audit_uuid=audit['uuid'])
         action_plan = action_plans['action_plans'][0]
 
         _, action_plan = self.client.show_action_plan(action_plan['uuid'])
@@ -75,7 +77,8 @@ class TestCreateDeleteExecuteActionPlan(base.BaseInfraOptimTest):
             duration=30,
             sleep_for=.5
         ))
-        _, action_plans = self.client.list_action_plan_by_audit(audit['uuid'])
+        _, action_plans = self.client.list_action_plans(
+            audit_uuid=audit['uuid'])
         action_plan = action_plans['action_plans'][0]
 
         _, action_plan = self.client.show_action_plan(action_plan['uuid'])
@@ -112,8 +115,8 @@ class TestShowListActionPlan(base.BaseInfraOptimTest):
             duration=30,
             sleep_for=.5
         )
-        _, action_plans = cls.client.list_action_plan_by_audit(
+        _, action_plans = cls.client.list_action_plans(
-            cls.audit['uuid'])
+            audit_uuid=cls.audit['uuid'])
         cls.action_plan = action_plans['action_plans'][0]
 
     @test.attr(type='smoke')
@@ -155,7 +158,7 @@ class TestShowListActionPlan(base.BaseInfraOptimTest):
     def test_list_with_limit(self):
         # We create 3 extra audits to exceed the limit we fix
         for _ in range(3):
-            self.create_audit(self.audit_template['uuid'])
+            self.create_action_plan(self.audit_template['uuid'])
 
         _, body = self.client.list_action_plans(limit=3)
 

watcher_tempest_plugin/tests/scenario/test_execute_basic_optim.py

@@ -117,7 +117,8 @@ class TestExecuteBasicStrategy(base.BaseInfraOptimScenarioTest):
             duration=300,
             sleep_for=2
         ))
-        _, action_plans = self.client.list_action_plan_by_audit(audit['uuid'])
+        _, action_plans = self.client.list_action_plans(
+            audit_uuid=audit['uuid'])
         action_plan = action_plans['action_plans'][0]
 
         _, action_plan = self.client.show_action_plan(action_plan['uuid'])

watcher_tempest_plugin/tests/scenario/test_execute_dummy_optim.py

@@ -45,7 +45,8 @@ class TestExecuteDummyStrategy(base.BaseInfraOptimScenarioTest):
             duration=30,
             sleep_for=.5
         ))
-        _, action_plans = self.client.list_action_plan_by_audit(audit['uuid'])
+        _, action_plans = self.client.list_action_plans(
+            audit_uuid=audit['uuid'])
         action_plan = action_plans['action_plans'][0]
 
         _, action_plan = self.client.show_action_plan(action_plan['uuid'])