Change file loading mechanism for heat tests

Improve file loading mechanism for heat tests. Previously,
heat tests loaded template files for every tests. "types.set"
allows to exclude redundant file operations from tests.
In addition, the unit tests for heat rally-scenarios were updated
and mocking of CONF.benchmark was excluded from unit-tests.

Change-Id: Ibdace7bf96ff822701ee8505f1dcd232999a6fe8

rally-jobs/extra/default.yaml.template

@@ -0,0 +1 @@
+heat_template_version: 2014-10-16

rally-jobs/rally.yaml

@@ -571,6 +571,8 @@
 
   HeatStacks.create_and_list_stack:
     -
+      args:
+        template_path: "/home/jenkins/.rally/extra/default.yaml.template"
       runner:
         type: "constant"
         times: 6
@@ -585,6 +587,8 @@
 
   HeatStacks.create_and_delete_stack:
     -
+      args:
+        template_path: "/home/jenkins/.rally/extra/default.yaml.template"
       runner:
         type: "constant"
         times: 6

rally/benchmark/scenarios/heat/stacks.py

@@ -15,6 +15,7 @@
 
 from rally.benchmark.scenarios import base
 from rally.benchmark.scenarios.heat import utils
+from rally.benchmark import types
 from rally.benchmark import validation
 from rally import consts
 
@@ -25,40 +26,26 @@ class HeatStacks(utils.HeatScenario):
     RESOURCE_NAME_PREFIX = "rally_stack_"
     RESOURCE_NAME_LENGTH = 7
 
-    @staticmethod
+    @types.set(template_path=types.FileType)
-    def _get_template_from_file(template_path):
-        template = None
-        if template_path:
-            try:
-                with open(template_path, "r") as f:
-                    template = f.read()
-            except IOError:
-                raise IOError("Provided path '%(template_path)s' is not valid"
-                              % {"template_path": template_path})
-        return template
-
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"]})
-    def create_and_list_stack(self, template_path=None):
+    def create_and_list_stack(self, template_path):
         """Add a stack and then list all stacks.
 
         Measure the "heat stack-create" and "heat stack-list" commands
         performance.
 
-        :param template_path: path to template file. If None or incorrect,
+        :param template_path: path to stack template file
-                              then default empty template will be used.
         """
-
+        self._create_stack(template_path)
-        template = self._get_template_from_file(template_path)
-        self._create_stack(template)
         self._list_stacks()
 
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario()
     def list_stacks_and_resources(self):
-        """List resources from tenant stacks."""
+        """List all resources from tenant stacks."""
 
         stacks = self._list_stacks()
         with base.AtomicAction(
@@ -66,27 +53,27 @@ class HeatStacks(utils.HeatScenario):
             for stack in stacks:
                 self.clients("heat").resources.list(stack.id)
 
+    @types.set(template_path=types.FileType)
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"]})
-    def create_and_delete_stack(self, template_path=None):
+    def create_and_delete_stack(self, template_path):
         """Add and then delete a stack.
 
         Measure the "heat stack-create" and "heat stack-delete" commands
         performance.
 
-        :param template_path: path to template file. If None or incorrect,
+        :param template_path: path to stack template file
-                              then default empty template will be used.
         """
 
-        template = self._get_template_from_file(template_path)
+        stack = self._create_stack(template_path)
-        stack = self._create_stack(template)
         self._delete_stack(stack)
 
+    @types.set(template_path=types.FileType)
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"]})
-    def create_check_delete_stack(self, template_path=None):
+    def create_check_delete_stack(self, template_path):
         """Create, check and delete a stack.
 
         Measure the performance of the following commands:
@@ -94,44 +81,37 @@ class HeatStacks(utils.HeatScenario):
         - heat action-check
         - heat stack-delete
 
-        :param template_path: path to template file that will be used for
+        :param template_path: path to stack template file
-                              stack create. If None or incorrect, then
-                              default empty template will be used.
         """
 
-        template = self._get_template_from_file(template_path)
+        stack = self._create_stack(template_path)
-        stack = self._create_stack(template)
         self._check_stack(stack)
         self._delete_stack(stack)
 
+    @types.set(template_path=types.FileType,
+               updated_template_path=types.FileType)
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"]})
-    def create_update_delete_stack(self, template_path=None,
+    def create_update_delete_stack(self, template_path, updated_template_path):
-                                   updated_template_path=None):
         """Add, update and then delete a stack.
 
         Measure the "heat stack-create", "heat stack-update"
         and "heat stack-delete" commands performance.
 
-        :param template_path: path to template file. If None or incorrect,
+        :param template_path: path to stack template file
-                              then default empty template will be used.
+        :param updated_template_path: path to updated stack template file
-        :param updated_template_path: path to template file that will be used
-                                      for stack update. If None or incorrect,
-                                      then default empty template will be
-                                      used instead.
         """
 
-        template = self._get_template_from_file(template_path)
+        stack = self._create_stack(template_path)
-        stack = self._create_stack(template)
+        self._update_stack(stack, updated_template_path)
-        updated_template = self._get_template_from_file(updated_template_path)
-        self._update_stack(stack, updated_template)
         self._delete_stack(stack)
 
+    @types.set(template_path=types.FileType)
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @base.scenario(context={"cleanup": ["heat"]})
-    def create_suspend_resume_delete_stack(self, template_path=None):
+    def create_suspend_resume_delete_stack(self, template_path):
         """Create, suspend-resume and then delete a stack.
 
         Measure performance of the following commands:
@@ -140,12 +120,10 @@ class HeatStacks(utils.HeatScenario):
         heat action-resume
         heat stack-delete
 
-        :param template_path: path to template file. If None or incorrect,
+        :param template_path: path to stack template file
-                              then default empty template will be used.
         """
 
-        template = self._get_template_from_file(template_path)
+        s = self._create_stack(template_path)
-        s = self._create_stack(template)
         self._suspend_stack(s)
         self._resume_stack(s)
         self._delete_stack(s)

rally/benchmark/scenarios/heat/utils.py

@@ -82,8 +82,6 @@ CONF.register_opts(HEAT_BENCHMARK_OPTS, group=benchmark_group)
 class HeatScenario(base.Scenario):
     """Base class for Heat scenarios with basic atomic actions."""
 
-    default_template = "HeatTemplateFormatVersion: '2012-12-12'"
-
     @base.atomic_action_timer("heat.list_stacks")
     def _list_stacks(self):
         """Return user stack list."""
@@ -91,15 +89,14 @@ class HeatScenario(base.Scenario):
         return list(self.clients("heat").stacks.list())
 
     @base.atomic_action_timer("heat.create_stack")
-    def _create_stack(self, template=None):
+    def _create_stack(self, template):
         """Create a new stack.
 
-        :param template: optional parameter. Template with stack description.
+        :param template: template with stack description.
 
         :returns: object of stack
         """
         stack_name = self._generate_random_name()
-        template = template or self.default_template
         kw = {
             "stack_name": stack_name,
             "disable_rollback": True,
@@ -126,7 +123,7 @@ class HeatScenario(base.Scenario):
         return stack
 
     @base.atomic_action_timer("heat.update_stack")
-    def _update_stack(self, stack, template=None):
+    def _update_stack(self, stack, template):
         """Update an existing stack
 
         :param stack: stack that need to be updated
@@ -134,8 +131,6 @@ class HeatScenario(base.Scenario):
         :returns: object of updated stack
         """
 
-        template = template or self.default_template
-
         kw = {
             "stack_name": stack.stack_name,
             "disable_rollback": True,

samples/tasks/scenarios/heat/create-and-delete-stack.json

@@ -1,6 +1,9 @@
 {
     "HeatStacks.create_and_delete_stack": [
         {
+            "args": {
+               "template_path": "templates/default.yaml.template"
+            },
             "runner": {
                 "type": "constant",
                 "times": 10,

samples/tasks/scenarios/heat/create-and-delete-stack.yaml

@@ -1,6 +1,8 @@
 ---
   HeatStacks.create_and_delete_stack:
     -
+      args:
+        template_path: "templates/default.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-and-list-stack.json

@@ -1,6 +1,9 @@
 {
     "HeatStacks.create_and_list_stack": [
         {
+            "args": {
+               "template_path": "templates/default.yaml.template"
+            },
             "runner": {
                 "type": "constant",
                 "times": 10,

samples/tasks/scenarios/heat/create-and-list-stack.yaml

@@ -1,6 +1,8 @@
 ---
   HeatStacks.create_and_list_stack:
     -
+      args:
+        template_path: "templates/default.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/templates/default.yaml.template

@@ -0,0 +1 @@
+heat_template_version: 2014-10-16

tests/unit/benchmark/scenarios/heat/test_stacks.py

@@ -13,8 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import tempfile
-
 import mock
 
 from rally.benchmark.scenarios.heat import stacks
@@ -25,16 +23,20 @@ HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks"
 
 class HeatStacksTestCase(test.TestCase):
 
+    def setUp(self):
+        super(HeatStacksTestCase, self).setUp()
+        self.default_template = "heat_template_version: 2013-05-23"
+
     @mock.patch(HEAT_STACKS + "._generate_random_name")
     @mock.patch(HEAT_STACKS + "._list_stacks")
     @mock.patch(HEAT_STACKS + "._create_stack")
     def test_create_and_list_stack(self, mock_create, mock_list,
                                    mock_random_name):
-        template_file = tempfile.NamedTemporaryFile()
         heat_scenario = stacks.HeatStacks()
         mock_random_name.return_value = "test-rally-stack"
-        heat_scenario.create_and_list_stack(template_path=template_file.name)
+        heat_scenario.create_and_list_stack(
-        self.assertEqual(1, mock_create.called)
+            template_path=self.default_template)
+        mock_create.assert_called_once_with(self.default_template)
         mock_list.assert_called_once_with()
 
     @mock.patch(HEAT_STACKS + ".clients")
@@ -59,17 +61,6 @@ class HeatStacksTestCase(test.TestCase):
         self._test_atomic_action_timer(
             heat_scenario.atomic_actions(), "heat.list_events_of_1_stacks")
 
-    @mock.patch(HEAT_STACKS + "._generate_random_name")
-    @mock.patch(HEAT_STACKS + "._list_stacks")
-    @mock.patch(HEAT_STACKS + "._create_stack")
-    def test_create_and_list_stack_fails(self, mock_create, mock_list,
-                                         mock_random_name):
-        heat_scenario = stacks.HeatStacks()
-        mock_random_name.return_value = "test-rally-stack"
-        self.assertRaises(IOError,
-                          heat_scenario.create_and_list_stack,
-                          template_path="/tmp/dummy")
-
     @mock.patch(HEAT_STACKS + "._generate_random_name")
     @mock.patch(HEAT_STACKS + "._delete_stack")
     @mock.patch(HEAT_STACKS + "._create_stack")
@@ -79,9 +70,10 @@ class HeatStacksTestCase(test.TestCase):
         fake_stack = object()
         mock_create.return_value = fake_stack
         mock_random_name.return_value = "test-rally-stack"
-        heat_scenario.create_and_delete_stack()
+        heat_scenario.create_and_delete_stack(
+            template_path=self.default_template)
 
-        self.assertTrue(mock_create.called)
+        mock_create.assert_called_once_with(self.default_template)
         mock_delete.assert_called_once_with(fake_stack)
 
     @mock.patch(HEAT_STACKS + "._delete_stack")
@@ -91,8 +83,9 @@ class HeatStacksTestCase(test.TestCase):
                                        mock_delete):
         heat_scenario = stacks.HeatStacks()
         mock_create.return_value = "fake_stack_create_check_delete"
-        heat_scenario.create_check_delete_stack()
+        heat_scenario.create_check_delete_stack(
-        mock_create.assert_called_once_with(None)
+            template_path=self.default_template)
+        mock_create.assert_called_once_with(self.default_template)
         mock_check.assert_called_once_with("fake_stack_create_check_delete")
         mock_delete.assert_called_once_with("fake_stack_create_check_delete")
 
@@ -106,10 +99,12 @@ class HeatStacksTestCase(test.TestCase):
         fake_stack = object()
         mock_create.return_value = fake_stack
         mock_random_name.return_value = "test-rally-stack"
-        heat_scenario.create_update_delete_stack()
+        heat_scenario.create_update_delete_stack(
+            template_path=self.default_template,
+            updated_template_path=self.default_template)
 
-        self.assertTrue(mock_create.called)
+        mock_create.assert_called_once_with(self.default_template)
-        mock_update.assert_called_once_with(fake_stack, None)
+        mock_update.assert_called_once_with(fake_stack, self.default_template)
         mock_delete.assert_called_once_with(fake_stack)
 
     @mock.patch(HEAT_STACKS + "._delete_stack")
@@ -123,9 +118,10 @@ class HeatStacksTestCase(test.TestCase):
                                                 mock_delete):
         heat_scenario = stacks.HeatStacks()
         mock_create.return_value = "fake_stack_create_suspend_resume_delete"
-        heat_scenario.create_suspend_resume_delete_stack()
+        heat_scenario.create_suspend_resume_delete_stack(
+            template_path=self.default_template)
 
-        mock_create.assert_called_once_with(None)
+        mock_create.assert_called_once_with(self.default_template)
         mock_suspend.assert_called_once_with(
             "fake_stack_create_suspend_resume_delete")
         mock_resume.assert_called_once_with(

tests/unit/benchmark/scenarios/heat/test_utils.py

@@ -23,6 +23,8 @@ from tests.unit import test
 BM_UTILS = "rally.benchmark.utils"
 HEAT_UTILS = "rally.benchmark.scenarios.heat.utils"
 
+CONF = utils.CONF
+
 
 class HeatScenarioTestCase(test.TestCase):
     def setUp(self):
@@ -40,6 +42,7 @@ class HeatScenarioTestCase(test.TestCase):
         self.gfm = self.get_fm.mock
         self.useFixture(mockpatch.Patch("time.sleep"))
         self.scenario = utils.HeatScenario()
+        self.default_template = "heat_template_version: 2013-05-23"
 
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
     def test_list_stacks(self, mock_clients):
@@ -58,32 +61,29 @@ class HeatScenarioTestCase(test.TestCase):
         }
         mock_clients("heat").stacks.get.return_value = self.stack
         scenario = utils.HeatScenario()
-        return_stack = scenario._create_stack()
+        return_stack = scenario._create_stack(self.default_template)
-        self.wait_for.mock.assert_called_once_with(self.stack,
+        self.wait_for.mock.assert_called_once_with(
-                                                   update_resource=self.gfm(),
+            self.stack,
-                                                   is_ready=self.res_is.mock(),
+            update_resource=self.gfm(),
-                                                   check_interval=1,
+            is_ready=self.res_is.mock(),
-                                                   timeout=3600)
+            check_interval=CONF.benchmark.heat_stack_create_poll_interval,
+            timeout=CONF.benchmark.heat_stack_create_timeout)
         self.res_is.mock.assert_has_calls([mock.call("CREATE_COMPLETE")])
         self.assertEqual(self.wait_for.mock(), return_stack)
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.create_stack")
 
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
-    @mock.patch(HEAT_UTILS + ".CONF.benchmark")
+    def test_update_stack(self, mock_clients):
-    def test_update_stack(self, mock_bench, mock_clients):
         mock_clients("heat").stacks.update.return_value = None
-        mock_bench.heat_stack_update_timeout = 1
-        mock_bench.heat_stack_update_poll_interval = 1
-
         scenario = utils.HeatScenario()
-        scenario._update_stack(self.stack, None)
+        scenario._update_stack(self.stack, self.default_template)
         self.wait_for.mock.assert_called_once_with(
             self.stack,
             update_resource=self.gfm(),
             is_ready=self.res_is.mock(),
-            check_interval=1,
+            check_interval=CONF.benchmark.heat_stack_update_poll_interval,
-            timeout=1)
+            timeout=CONF.benchmark.heat_stack_update_timeout)
         self.res_is.mock.assert_has_calls([mock.call("UPDATE_COMPLETE")])
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.update_stack")
@@ -98,8 +98,8 @@ class HeatScenarioTestCase(test.TestCase):
             self.stack,
             update_resource=self.gfm(),
             is_ready=self.res_is.mock(),
-            check_interval=utils.CONF.benchmark.heat_stack_check_poll_interval,
+            check_interval=CONF.benchmark.heat_stack_check_poll_interval,
-            timeout=utils.CONF.benchmark.heat_stack_check_timeout)
+            timeout=CONF.benchmark.heat_stack_check_timeout)
         self.res_is.mock.assert_has_calls([mock.call("CHECK_COMPLETE")])
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.check_stack")
@@ -111,17 +111,13 @@ class HeatScenarioTestCase(test.TestCase):
         self.wait_for_delete.mock.assert_called_once_with(
             self.stack,
             update_resource=self.gfm(),
-            check_interval=1,
+            check_interval=CONF.benchmark.heat_stack_delete_poll_interval,
-            timeout=3600)
+            timeout=CONF.benchmark.heat_stack_delete_timeout)
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.delete_stack")
 
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
-    @mock.patch(HEAT_UTILS + ".CONF.benchmark")
+    def test_suspend_stack(self, mock_clients):
-    def test_suspend_stack(self, mock_bench, mock_clients):
-        mock_bench.heat_stack_suspend_timeout = 1
-        mock_bench.heat_stack_suspend_poll_interval = 1
-
         scenario = utils.HeatScenario()
         scenario._suspend_stack(self.stack)
         mock_clients("heat").actions.suspend.assert_called_once_with(
@@ -130,18 +126,14 @@ class HeatScenarioTestCase(test.TestCase):
             self.stack,
             update_resource=self.gfm(),
             is_ready=self.res_is.mock(),
-            check_interval=1,
+            check_interval=CONF.benchmark.heat_stack_suspend_poll_interval,
-            timeout=1)
+            timeout=CONF.benchmark.heat_stack_suspend_timeout)
         self.res_is.mock.assert_has_calls([mock.call("SUSPEND_COMPLETE")])
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.suspend_stack")
 
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
-    @mock.patch(HEAT_UTILS + ".CONF.benchmark")
+    def test_resume_stack(self, mock_clients):
-    def test_resume_stack(self, mock_bench, mock_clients):
-        mock_bench.heat_stack_resume_timeout = 1
-        mock_bench.heat_stack_resume_poll_interval = 1
-
         scenario = utils.HeatScenario()
         scenario._resume_stack(self.stack)
         mock_clients("heat").actions.resume.assert_called_once_with(
@@ -150,8 +142,8 @@ class HeatScenarioTestCase(test.TestCase):
             self.stack,
             update_resource=self.gfm(),
             is_ready=self.res_is.mock(),
-            check_interval=1,
+            check_interval=CONF.benchmark.heat_stack_resume_poll_interval,
-            timeout=1)
+            timeout=CONF.benchmark.heat_stack_resume_timeout)
         self.res_is.mock.assert_has_calls([mock.call("RESUME_COMPLETE")])
         self._test_atomic_action_timer(scenario.atomic_actions(),
                                        "heat.resume_stack")
@@ -159,12 +151,7 @@ class HeatScenarioTestCase(test.TestCase):
 
 class HeatScenarioNegativeTestCase(test.TestCase):
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
-    @mock.patch(HEAT_UTILS + ".CONF.benchmark")
+    def test_failed_create_stack(self, mock_clients):
-    def test_failed_create_stack(self, mock_bench, mock_clients):
-        mock_bench.heat_stack_create_prepoll_delay = 2
-        mock_bench.heat_stack_create_timeout = 1
-        mock_bench.benchmark.heat_stack_create_poll_interval = 1
-
         mock_clients("heat").stacks.create.return_value = {
             "stack": {"id": "test_id"}
         }
@@ -182,12 +169,7 @@ class HeatScenarioNegativeTestCase(test.TestCase):
             raise self.fail("Unrecognized error status")
 
     @mock.patch(HEAT_UTILS + ".HeatScenario.clients")
-    @mock.patch(HEAT_UTILS + ".CONF.benchmark")
+    def test_failed_update_stack(self, mock_clients):
-    def test_failed_update_stack(self, mock_bench, mock_clients):
-        mock_bench.heat_stack_update_prepoll_delay = 2
-        mock_bench.heat_stack_update_timeout = 1
-        mock_bench.benchmark.heat_stack_update_poll_interval = 1
-
         stack = mock.Mock()
         resource = mock.Mock()
         resource.stack_status = "UPDATE_FAILED"
@@ -196,7 +178,8 @@ class HeatScenarioNegativeTestCase(test.TestCase):
         scenario = utils.HeatScenario()
         try:
             ex = self.assertRaises(exceptions.GetResourceErrorStatus,
-                                   scenario._update_stack, stack)
+                                   scenario._update_stack, stack,
+                                   "heat_template_version: 2013-05-23")
             self.assertIn("has UPDATE_FAILED status", str(ex))
         except exceptions.TimeoutException:
             raise self.fail("Unrecognized error status")