diff --git a/rally-jobs/extra/default.yaml.template b/rally-jobs/extra/default.yaml.template new file mode 100644 index 0000000000..eb4f2f2dd8 --- /dev/null +++ b/rally-jobs/extra/default.yaml.template @@ -0,0 +1 @@ +heat_template_version: 2014-10-16 \ No newline at end of file diff --git a/rally-jobs/rally.yaml b/rally-jobs/rally.yaml index 3a5c185508..3efe0a24c0 100755 --- a/rally-jobs/rally.yaml +++ b/rally-jobs/rally.yaml @@ -571,6 +571,8 @@ HeatStacks.create_and_list_stack: - + args: + template_path: "/home/jenkins/.rally/extra/default.yaml.template" runner: type: "constant" times: 6 @@ -585,6 +587,8 @@ HeatStacks.create_and_delete_stack: - + args: + template_path: "/home/jenkins/.rally/extra/default.yaml.template" runner: type: "constant" times: 6 diff --git a/rally/benchmark/scenarios/heat/stacks.py b/rally/benchmark/scenarios/heat/stacks.py index 468868656e..bb3328e445 100644 --- a/rally/benchmark/scenarios/heat/stacks.py +++ b/rally/benchmark/scenarios/heat/stacks.py @@ -15,6 +15,7 @@ from rally.benchmark.scenarios import base from rally.benchmark.scenarios.heat import utils +from rally.benchmark import types from rally.benchmark import validation from rally import consts @@ -25,40 +26,26 @@ class HeatStacks(utils.HeatScenario): RESOURCE_NAME_PREFIX = "rally_stack_" RESOURCE_NAME_LENGTH = 7 - @staticmethod - def _get_template_from_file(template_path): - template = None - if template_path: - try: - with open(template_path, "r") as f: - template = f.read() - except IOError: - raise IOError("Provided path '%(template_path)s' is not valid" - % {"template_path": template_path}) - return template - + @types.set(template_path=types.FileType) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario(context={"cleanup": ["heat"]}) - def create_and_list_stack(self, template_path=None): + def create_and_list_stack(self, template_path): """Add a stack and then list all stacks. Measure the "heat stack-create" and "heat stack-list" commands performance. - :param template_path: path to template file. If None or incorrect, - then default empty template will be used. + :param template_path: path to stack template file """ - - template = self._get_template_from_file(template_path) - self._create_stack(template) + self._create_stack(template_path) self._list_stacks() @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario() def list_stacks_and_resources(self): - """List resources from tenant stacks.""" + """List all resources from tenant stacks.""" stacks = self._list_stacks() with base.AtomicAction( @@ -66,27 +53,27 @@ class HeatStacks(utils.HeatScenario): for stack in stacks: self.clients("heat").resources.list(stack.id) + @types.set(template_path=types.FileType) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario(context={"cleanup": ["heat"]}) - def create_and_delete_stack(self, template_path=None): + def create_and_delete_stack(self, template_path): """Add and then delete a stack. Measure the "heat stack-create" and "heat stack-delete" commands performance. - :param template_path: path to template file. If None or incorrect, - then default empty template will be used. + :param template_path: path to stack template file """ - template = self._get_template_from_file(template_path) - stack = self._create_stack(template) + stack = self._create_stack(template_path) self._delete_stack(stack) + @types.set(template_path=types.FileType) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario(context={"cleanup": ["heat"]}) - def create_check_delete_stack(self, template_path=None): + def create_check_delete_stack(self, template_path): """Create, check and delete a stack. Measure the performance of the following commands: @@ -94,44 +81,37 @@ class HeatStacks(utils.HeatScenario): - heat action-check - heat stack-delete - :param template_path: path to template file that will be used for - stack create. If None or incorrect, then - default empty template will be used. + :param template_path: path to stack template file """ - template = self._get_template_from_file(template_path) - stack = self._create_stack(template) + stack = self._create_stack(template_path) self._check_stack(stack) self._delete_stack(stack) + @types.set(template_path=types.FileType, + updated_template_path=types.FileType) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario(context={"cleanup": ["heat"]}) - def create_update_delete_stack(self, template_path=None, - updated_template_path=None): + def create_update_delete_stack(self, template_path, updated_template_path): """Add, update and then delete a stack. Measure the "heat stack-create", "heat stack-update" and "heat stack-delete" commands performance. - :param template_path: path to template file. If None or incorrect, - then default empty template will be used. - :param updated_template_path: path to template file that will be used - for stack update. If None or incorrect, - then default empty template will be - used instead. + :param template_path: path to stack template file + :param updated_template_path: path to updated stack template file """ - template = self._get_template_from_file(template_path) - stack = self._create_stack(template) - updated_template = self._get_template_from_file(updated_template_path) - self._update_stack(stack, updated_template) + stack = self._create_stack(template_path) + self._update_stack(stack, updated_template_path) self._delete_stack(stack) + @types.set(template_path=types.FileType) @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @base.scenario(context={"cleanup": ["heat"]}) - def create_suspend_resume_delete_stack(self, template_path=None): + def create_suspend_resume_delete_stack(self, template_path): """Create, suspend-resume and then delete a stack. Measure performance of the following commands: @@ -140,12 +120,10 @@ class HeatStacks(utils.HeatScenario): heat action-resume heat stack-delete - :param template_path: path to template file. If None or incorrect, - then default empty template will be used. + :param template_path: path to stack template file """ - template = self._get_template_from_file(template_path) - s = self._create_stack(template) + s = self._create_stack(template_path) self._suspend_stack(s) self._resume_stack(s) self._delete_stack(s) diff --git a/rally/benchmark/scenarios/heat/utils.py b/rally/benchmark/scenarios/heat/utils.py index b544733c1c..8480878311 100644 --- a/rally/benchmark/scenarios/heat/utils.py +++ b/rally/benchmark/scenarios/heat/utils.py @@ -82,8 +82,6 @@ CONF.register_opts(HEAT_BENCHMARK_OPTS, group=benchmark_group) class HeatScenario(base.Scenario): """Base class for Heat scenarios with basic atomic actions.""" - default_template = "HeatTemplateFormatVersion: '2012-12-12'" - @base.atomic_action_timer("heat.list_stacks") def _list_stacks(self): """Return user stack list.""" @@ -91,15 +89,14 @@ class HeatScenario(base.Scenario): return list(self.clients("heat").stacks.list()) @base.atomic_action_timer("heat.create_stack") - def _create_stack(self, template=None): + def _create_stack(self, template): """Create a new stack. - :param template: optional parameter. Template with stack description. + :param template: template with stack description. :returns: object of stack """ stack_name = self._generate_random_name() - template = template or self.default_template kw = { "stack_name": stack_name, "disable_rollback": True, @@ -126,7 +123,7 @@ class HeatScenario(base.Scenario): return stack @base.atomic_action_timer("heat.update_stack") - def _update_stack(self, stack, template=None): + def _update_stack(self, stack, template): """Update an existing stack :param stack: stack that need to be updated @@ -134,8 +131,6 @@ class HeatScenario(base.Scenario): :returns: object of updated stack """ - template = template or self.default_template - kw = { "stack_name": stack.stack_name, "disable_rollback": True, diff --git a/samples/tasks/scenarios/heat/create-and-delete-stack.json b/samples/tasks/scenarios/heat/create-and-delete-stack.json index cc897ab5c7..0887b18007 100644 --- a/samples/tasks/scenarios/heat/create-and-delete-stack.json +++ b/samples/tasks/scenarios/heat/create-and-delete-stack.json @@ -1,6 +1,9 @@ { "HeatStacks.create_and_delete_stack": [ { + "args": { + "template_path": "templates/default.yaml.template" + }, "runner": { "type": "constant", "times": 10, diff --git a/samples/tasks/scenarios/heat/create-and-delete-stack.yaml b/samples/tasks/scenarios/heat/create-and-delete-stack.yaml index f66f88d116..f6de6a9a55 100644 --- a/samples/tasks/scenarios/heat/create-and-delete-stack.yaml +++ b/samples/tasks/scenarios/heat/create-and-delete-stack.yaml @@ -1,6 +1,8 @@ --- HeatStacks.create_and_delete_stack: - + args: + template_path: "templates/default.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-and-list-stack.json b/samples/tasks/scenarios/heat/create-and-list-stack.json index a013542da1..961efd52bc 100644 --- a/samples/tasks/scenarios/heat/create-and-list-stack.json +++ b/samples/tasks/scenarios/heat/create-and-list-stack.json @@ -1,6 +1,9 @@ { "HeatStacks.create_and_list_stack": [ { + "args": { + "template_path": "templates/default.yaml.template" + }, "runner": { "type": "constant", "times": 10, diff --git a/samples/tasks/scenarios/heat/create-and-list-stack.yaml b/samples/tasks/scenarios/heat/create-and-list-stack.yaml index 323fe9152c..61a1b34508 100644 --- a/samples/tasks/scenarios/heat/create-and-list-stack.yaml +++ b/samples/tasks/scenarios/heat/create-and-list-stack.yaml @@ -1,6 +1,8 @@ --- HeatStacks.create_and_list_stack: - + args: + template_path: "templates/default.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/templates/default.yaml.template b/samples/tasks/scenarios/heat/templates/default.yaml.template new file mode 100644 index 0000000000..eb4f2f2dd8 --- /dev/null +++ b/samples/tasks/scenarios/heat/templates/default.yaml.template @@ -0,0 +1 @@ +heat_template_version: 2014-10-16 \ No newline at end of file diff --git a/tests/unit/benchmark/scenarios/heat/test_stacks.py b/tests/unit/benchmark/scenarios/heat/test_stacks.py index 2abb6ebc12..a256ac6a1b 100644 --- a/tests/unit/benchmark/scenarios/heat/test_stacks.py +++ b/tests/unit/benchmark/scenarios/heat/test_stacks.py @@ -13,8 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import tempfile - import mock from rally.benchmark.scenarios.heat import stacks @@ -25,16 +23,20 @@ HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks" class HeatStacksTestCase(test.TestCase): + def setUp(self): + super(HeatStacksTestCase, self).setUp() + self.default_template = "heat_template_version: 2013-05-23" + @mock.patch(HEAT_STACKS + "._generate_random_name") @mock.patch(HEAT_STACKS + "._list_stacks") @mock.patch(HEAT_STACKS + "._create_stack") def test_create_and_list_stack(self, mock_create, mock_list, mock_random_name): - template_file = tempfile.NamedTemporaryFile() heat_scenario = stacks.HeatStacks() mock_random_name.return_value = "test-rally-stack" - heat_scenario.create_and_list_stack(template_path=template_file.name) - self.assertEqual(1, mock_create.called) + heat_scenario.create_and_list_stack( + template_path=self.default_template) + mock_create.assert_called_once_with(self.default_template) mock_list.assert_called_once_with() @mock.patch(HEAT_STACKS + ".clients") @@ -59,17 +61,6 @@ class HeatStacksTestCase(test.TestCase): self._test_atomic_action_timer( heat_scenario.atomic_actions(), "heat.list_events_of_1_stacks") - @mock.patch(HEAT_STACKS + "._generate_random_name") - @mock.patch(HEAT_STACKS + "._list_stacks") - @mock.patch(HEAT_STACKS + "._create_stack") - def test_create_and_list_stack_fails(self, mock_create, mock_list, - mock_random_name): - heat_scenario = stacks.HeatStacks() - mock_random_name.return_value = "test-rally-stack" - self.assertRaises(IOError, - heat_scenario.create_and_list_stack, - template_path="/tmp/dummy") - @mock.patch(HEAT_STACKS + "._generate_random_name") @mock.patch(HEAT_STACKS + "._delete_stack") @mock.patch(HEAT_STACKS + "._create_stack") @@ -79,9 +70,10 @@ class HeatStacksTestCase(test.TestCase): fake_stack = object() mock_create.return_value = fake_stack mock_random_name.return_value = "test-rally-stack" - heat_scenario.create_and_delete_stack() + heat_scenario.create_and_delete_stack( + template_path=self.default_template) - self.assertTrue(mock_create.called) + mock_create.assert_called_once_with(self.default_template) mock_delete.assert_called_once_with(fake_stack) @mock.patch(HEAT_STACKS + "._delete_stack") @@ -91,8 +83,9 @@ class HeatStacksTestCase(test.TestCase): mock_delete): heat_scenario = stacks.HeatStacks() mock_create.return_value = "fake_stack_create_check_delete" - heat_scenario.create_check_delete_stack() - mock_create.assert_called_once_with(None) + heat_scenario.create_check_delete_stack( + template_path=self.default_template) + mock_create.assert_called_once_with(self.default_template) mock_check.assert_called_once_with("fake_stack_create_check_delete") mock_delete.assert_called_once_with("fake_stack_create_check_delete") @@ -106,10 +99,12 @@ class HeatStacksTestCase(test.TestCase): fake_stack = object() mock_create.return_value = fake_stack mock_random_name.return_value = "test-rally-stack" - heat_scenario.create_update_delete_stack() + heat_scenario.create_update_delete_stack( + template_path=self.default_template, + updated_template_path=self.default_template) - self.assertTrue(mock_create.called) - mock_update.assert_called_once_with(fake_stack, None) + mock_create.assert_called_once_with(self.default_template) + mock_update.assert_called_once_with(fake_stack, self.default_template) mock_delete.assert_called_once_with(fake_stack) @mock.patch(HEAT_STACKS + "._delete_stack") @@ -123,9 +118,10 @@ class HeatStacksTestCase(test.TestCase): mock_delete): heat_scenario = stacks.HeatStacks() mock_create.return_value = "fake_stack_create_suspend_resume_delete" - heat_scenario.create_suspend_resume_delete_stack() + heat_scenario.create_suspend_resume_delete_stack( + template_path=self.default_template) - mock_create.assert_called_once_with(None) + mock_create.assert_called_once_with(self.default_template) mock_suspend.assert_called_once_with( "fake_stack_create_suspend_resume_delete") mock_resume.assert_called_once_with( diff --git a/tests/unit/benchmark/scenarios/heat/test_utils.py b/tests/unit/benchmark/scenarios/heat/test_utils.py index 21e1f11e25..3887fb46ab 100644 --- a/tests/unit/benchmark/scenarios/heat/test_utils.py +++ b/tests/unit/benchmark/scenarios/heat/test_utils.py @@ -23,6 +23,8 @@ from tests.unit import test BM_UTILS = "rally.benchmark.utils" HEAT_UTILS = "rally.benchmark.scenarios.heat.utils" +CONF = utils.CONF + class HeatScenarioTestCase(test.TestCase): def setUp(self): @@ -40,6 +42,7 @@ class HeatScenarioTestCase(test.TestCase): self.gfm = self.get_fm.mock self.useFixture(mockpatch.Patch("time.sleep")) self.scenario = utils.HeatScenario() + self.default_template = "heat_template_version: 2013-05-23" @mock.patch(HEAT_UTILS + ".HeatScenario.clients") def test_list_stacks(self, mock_clients): @@ -58,32 +61,29 @@ class HeatScenarioTestCase(test.TestCase): } mock_clients("heat").stacks.get.return_value = self.stack scenario = utils.HeatScenario() - return_stack = scenario._create_stack() - self.wait_for.mock.assert_called_once_with(self.stack, - update_resource=self.gfm(), - is_ready=self.res_is.mock(), - check_interval=1, - timeout=3600) + return_stack = scenario._create_stack(self.default_template) + self.wait_for.mock.assert_called_once_with( + self.stack, + update_resource=self.gfm(), + is_ready=self.res_is.mock(), + check_interval=CONF.benchmark.heat_stack_create_poll_interval, + timeout=CONF.benchmark.heat_stack_create_timeout) self.res_is.mock.assert_has_calls([mock.call("CREATE_COMPLETE")]) self.assertEqual(self.wait_for.mock(), return_stack) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.create_stack") @mock.patch(HEAT_UTILS + ".HeatScenario.clients") - @mock.patch(HEAT_UTILS + ".CONF.benchmark") - def test_update_stack(self, mock_bench, mock_clients): + def test_update_stack(self, mock_clients): mock_clients("heat").stacks.update.return_value = None - mock_bench.heat_stack_update_timeout = 1 - mock_bench.heat_stack_update_poll_interval = 1 - scenario = utils.HeatScenario() - scenario._update_stack(self.stack, None) + scenario._update_stack(self.stack, self.default_template) self.wait_for.mock.assert_called_once_with( self.stack, update_resource=self.gfm(), is_ready=self.res_is.mock(), - check_interval=1, - timeout=1) + check_interval=CONF.benchmark.heat_stack_update_poll_interval, + timeout=CONF.benchmark.heat_stack_update_timeout) self.res_is.mock.assert_has_calls([mock.call("UPDATE_COMPLETE")]) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.update_stack") @@ -98,8 +98,8 @@ class HeatScenarioTestCase(test.TestCase): self.stack, update_resource=self.gfm(), is_ready=self.res_is.mock(), - check_interval=utils.CONF.benchmark.heat_stack_check_poll_interval, - timeout=utils.CONF.benchmark.heat_stack_check_timeout) + check_interval=CONF.benchmark.heat_stack_check_poll_interval, + timeout=CONF.benchmark.heat_stack_check_timeout) self.res_is.mock.assert_has_calls([mock.call("CHECK_COMPLETE")]) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.check_stack") @@ -111,17 +111,13 @@ class HeatScenarioTestCase(test.TestCase): self.wait_for_delete.mock.assert_called_once_with( self.stack, update_resource=self.gfm(), - check_interval=1, - timeout=3600) + check_interval=CONF.benchmark.heat_stack_delete_poll_interval, + timeout=CONF.benchmark.heat_stack_delete_timeout) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.delete_stack") @mock.patch(HEAT_UTILS + ".HeatScenario.clients") - @mock.patch(HEAT_UTILS + ".CONF.benchmark") - def test_suspend_stack(self, mock_bench, mock_clients): - mock_bench.heat_stack_suspend_timeout = 1 - mock_bench.heat_stack_suspend_poll_interval = 1 - + def test_suspend_stack(self, mock_clients): scenario = utils.HeatScenario() scenario._suspend_stack(self.stack) mock_clients("heat").actions.suspend.assert_called_once_with( @@ -130,18 +126,14 @@ class HeatScenarioTestCase(test.TestCase): self.stack, update_resource=self.gfm(), is_ready=self.res_is.mock(), - check_interval=1, - timeout=1) + check_interval=CONF.benchmark.heat_stack_suspend_poll_interval, + timeout=CONF.benchmark.heat_stack_suspend_timeout) self.res_is.mock.assert_has_calls([mock.call("SUSPEND_COMPLETE")]) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.suspend_stack") @mock.patch(HEAT_UTILS + ".HeatScenario.clients") - @mock.patch(HEAT_UTILS + ".CONF.benchmark") - def test_resume_stack(self, mock_bench, mock_clients): - mock_bench.heat_stack_resume_timeout = 1 - mock_bench.heat_stack_resume_poll_interval = 1 - + def test_resume_stack(self, mock_clients): scenario = utils.HeatScenario() scenario._resume_stack(self.stack) mock_clients("heat").actions.resume.assert_called_once_with( @@ -150,8 +142,8 @@ class HeatScenarioTestCase(test.TestCase): self.stack, update_resource=self.gfm(), is_ready=self.res_is.mock(), - check_interval=1, - timeout=1) + check_interval=CONF.benchmark.heat_stack_resume_poll_interval, + timeout=CONF.benchmark.heat_stack_resume_timeout) self.res_is.mock.assert_has_calls([mock.call("RESUME_COMPLETE")]) self._test_atomic_action_timer(scenario.atomic_actions(), "heat.resume_stack") @@ -159,12 +151,7 @@ class HeatScenarioTestCase(test.TestCase): class HeatScenarioNegativeTestCase(test.TestCase): @mock.patch(HEAT_UTILS + ".HeatScenario.clients") - @mock.patch(HEAT_UTILS + ".CONF.benchmark") - def test_failed_create_stack(self, mock_bench, mock_clients): - mock_bench.heat_stack_create_prepoll_delay = 2 - mock_bench.heat_stack_create_timeout = 1 - mock_bench.benchmark.heat_stack_create_poll_interval = 1 - + def test_failed_create_stack(self, mock_clients): mock_clients("heat").stacks.create.return_value = { "stack": {"id": "test_id"} } @@ -182,12 +169,7 @@ class HeatScenarioNegativeTestCase(test.TestCase): raise self.fail("Unrecognized error status") @mock.patch(HEAT_UTILS + ".HeatScenario.clients") - @mock.patch(HEAT_UTILS + ".CONF.benchmark") - def test_failed_update_stack(self, mock_bench, mock_clients): - mock_bench.heat_stack_update_prepoll_delay = 2 - mock_bench.heat_stack_update_timeout = 1 - mock_bench.benchmark.heat_stack_update_poll_interval = 1 - + def test_failed_update_stack(self, mock_clients): stack = mock.Mock() resource = mock.Mock() resource.stack_status = "UPDATE_FAILED" @@ -196,7 +178,8 @@ class HeatScenarioNegativeTestCase(test.TestCase): scenario = utils.HeatScenario() try: ex = self.assertRaises(exceptions.GetResourceErrorStatus, - scenario._update_stack, stack) + scenario._update_stack, stack, + "heat_template_version: 2013-05-23") self.assertIn("has UPDATE_FAILED status", str(ex)) except exceptions.TimeoutException: raise self.fail("Unrecognized error status")