From 2fe25740bbf1a604890f945523fbb6f6fa213f48 Mon Sep 17 00:00:00 2001 From: kairat_kushaev Date: Wed, 4 Feb 2015 15:05:19 +0300 Subject: [PATCH] Add create-update-delete stack scenario The create-update-delete stack scenario executes the following actions with using Heat: - creates the new stack (with some resources) - updates the stack and stack resoruces with the new template - deletes the stack. Unit-tests for the changes above are also included into patch. Patch also adds rally test cases for stack create-update-delete stack scenario (because rally tests require test scenario to be used by test case samples): Case 1: - Heat creates a new stack - Heat updates the stack with adding additional resource to the stack - Heat deletes the stack Case 2: - Heat creates a stack - Heat updates the stack with deleting some resource from the stack - Heat deletes the stack. Change-Id: I4c0ff24d0651434450a281a62670f1bd6c2a0000 --- rally-jobs/extra/random_strings.yaml.template | 13 +++++ .../updated_random_strings_add.yaml.template | 19 +++++++ ...pdated_random_strings_delete.yaml.template | 11 ++++ rally-jobs/rally.yaml | 35 ++++++++++++- rally/benchmark/scenarios/heat/stacks.py | 38 +++++++++++--- rally/benchmark/scenarios/heat/utils.py | 50 +++++++++++++++++-- .../create-update-delete-stack-add-res.json | 21 ++++++++ .../create-update-delete-stack-add-res.yaml | 14 ++++++ .../create-update-delete-stack-del-res.json | 21 ++++++++ .../create-update-delete-stack-del-res.yaml | 14 ++++++ .../templates/random_strings.yaml.template | 13 +++++ .../updated_random_strings_add.yaml.template | 19 +++++++ ...pdated_random_strings_delete.yaml.template | 11 ++++ .../benchmark/scenarios/heat/test_stacks.py | 18 ++++++- .../benchmark/scenarios/heat/test_utils.py | 43 ++++++++++++++-- 15 files changed, 322 insertions(+), 18 deletions(-) create mode 100644 rally-jobs/extra/random_strings.yaml.template create mode 100644 rally-jobs/extra/updated_random_strings_add.yaml.template create mode 100644 rally-jobs/extra/updated_random_strings_delete.yaml.template create mode 100644 samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json create mode 100644 samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml create mode 100644 samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json create mode 100644 samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml create mode 100644 samples/tasks/scenarios/heat/templates/random_strings.yaml.template create mode 100644 samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template create mode 100644 samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template diff --git a/rally-jobs/extra/random_strings.yaml.template b/rally-jobs/extra/random_strings.yaml.template new file mode 100644 index 0000000000..2dd676c118 --- /dev/null +++ b/rally-jobs/extra/random_strings.yaml.template @@ -0,0 +1,13 @@ +heat_template_version: 2014-10-16 + +description: Test template for rally create-update-delete scenario + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_two: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/rally-jobs/extra/updated_random_strings_add.yaml.template b/rally-jobs/extra/updated_random_strings_add.yaml.template new file mode 100644 index 0000000000..e06d42e012 --- /dev/null +++ b/rally-jobs/extra/updated_random_strings_add.yaml.template @@ -0,0 +1,19 @@ +heat_template_version: 2014-10-16 + +description: > + Test template for create-update-delete-stack scenario in rally. + The template updates the stack defined by random_strings.yaml.template with additional resource. + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_two: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_three: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/rally-jobs/extra/updated_random_strings_delete.yaml.template b/rally-jobs/extra/updated_random_strings_delete.yaml.template new file mode 100644 index 0000000000..d02593e3b8 --- /dev/null +++ b/rally-jobs/extra/updated_random_strings_delete.yaml.template @@ -0,0 +1,11 @@ +heat_template_version: 2014-10-16 + +description: > + Test template for create-update-delete-stack scenario in rally. + The template deletes one resource from the stack defined by random_strings.yaml.template. + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/rally-jobs/rally.yaml b/rally-jobs/rally.yaml index 84a74171f3..bf26aa97c2 100755 --- a/rally-jobs/rally.yaml +++ b/rally-jobs/rally.yaml @@ -475,7 +475,7 @@ - args: - template_path: '/home/jenkins/.rally/extra/server_with_volume.yaml.template' + template_path: "/home/jenkins/.rally/extra/server_with_volume.yaml.template" runner: type: "constant" times: 2 @@ -488,6 +488,39 @@ failure_rate: max: 0 + HeatStacks.create_update_delete_stack: + - + args: + template_path: "/home/jenkins/.rally/extra/random_strings.yaml.template" + updated_template_path: "/home/jenkins/.rally/extra/updated_random_strings_add.yaml.template" + runner: + type: "constant" + times: 6 + concurrency: 3 + context: + users: + tenants: 2 + users_per_tenant: 3 + sla: + failure_rate: + max: 0 + + - + args: + template_path: "/home/jenkins/.rally/extra/random_strings.yaml.template" + updated_template_path: "/home/jenkins/.rally/extra/updated_random_strings_delete.yaml.template" + runner: + type: "constant" + times: 6 + concurrency: 3 + context: + users: + tenants: 2 + users_per_tenant: 3 + sla: + failure_rate: + max: 0 + Authenticate.keystone: - runner: diff --git a/rally/benchmark/scenarios/heat/stacks.py b/rally/benchmark/scenarios/heat/stacks.py index 880f14d857..2e8ef1311d 100644 --- a/rally/benchmark/scenarios/heat/stacks.py +++ b/rally/benchmark/scenarios/heat/stacks.py @@ -25,7 +25,8 @@ class HeatStacks(utils.HeatScenario): RESOURCE_NAME_PREFIX = "rally_stack_" RESOURCE_NAME_LENGTH = 7 - def _get_template_from_file(self, template_path): + @staticmethod + def _get_template_from_file(template_path): template = None if template_path: try: @@ -42,17 +43,15 @@ class HeatStacks(utils.HeatScenario): def create_and_list_stack(self, template_path=None): """Add a stack and then list all stacks. - Mesure the "heat stack-create" and "heat stack-list" commands + Measure the "heat stack-create" and "heat stack-list" commands performance. :param template_path: path to template file. If None or incorrect, then default empty template will be used. """ - stack_name = self._generate_random_name() template = self._get_template_from_file(template_path) - - self._create_stack(stack_name, template) + self._create_stack(template) self._list_stacks() @validation.required_services(consts.Service.HEAT) @@ -67,8 +66,31 @@ class HeatStacks(utils.HeatScenario): :param template_path: path to template file. If None or incorrect, then default empty template will be used. """ - stack_name = self._generate_random_name() - template = self._get_template_from_file(template_path) - stack = self._create_stack(stack_name, template) + template = self._get_template_from_file(template_path) + stack = self._create_stack(template) self._delete_stack(stack) + + @validation.required_services(consts.Service.HEAT) + @validation.required_openstack(users=True) + @base.scenario(context={"cleanup": ["heat"]}) + def create_update_delete_stack(self, template_path=None, + updated_template_path=None): + """Add, update and then delete a stack. + + Measure the "heat stack-create", "heat stack-update" + and "heat stack-delete" commands performance. + + :param template_path: path to template file. If None or incorrect, + then default empty template will be used. + :param updated_template_path: path to template file that will be used + for stack update. If None or incorrect, + then default empty template will be + used instead. + """ + + template = self._get_template_from_file(template_path) + stack = self._create_stack(template) + updated_template = self._get_template_from_file(updated_template_path) + self._update_stack(stack, updated_template) + self._delete_stack(stack) \ No newline at end of file diff --git a/rally/benchmark/scenarios/heat/utils.py b/rally/benchmark/scenarios/heat/utils.py index 1b49b27f27..29b731cb6f 100644 --- a/rally/benchmark/scenarios/heat/utils.py +++ b/rally/benchmark/scenarios/heat/utils.py @@ -39,7 +39,18 @@ heat_benchmark_opts = [ cfg.FloatOpt("heat_stack_delete_poll_interval", default=1.0, help="Interval between checks when waiting for stack " - "deletion.") + "deletion."), + cfg.FloatOpt("heat_stack_update_prepoll_delay", + default=2.0, + help="Time to sleep after updating a resource before " + "polling for it status"), + cfg.FloatOpt("heat_stack_update_timeout", + default=3600.0, + help="Time to wait for stack to be updated"), + cfg.FloatOpt("heat_stack_update_poll_interval", + default=1.0, + help="Interval between checks when waiting for stack " + "update."), ] @@ -60,16 +71,15 @@ class HeatScenario(base.Scenario): return list(self.clients("heat").stacks.list()) @base.atomic_action_timer("heat.create_stack") - def _create_stack(self, stack_name, template=None): + def _create_stack(self, template=None): """Create a new stack. - :param stack_name: string. Name for created stack. :param template: optional parameter. Template with stack description. :returns: object of stack """ + stack_name = self._generate_random_name() template = template or self.default_template - kw = { "stack_name": stack_name, "disable_rollback": True, @@ -95,6 +105,36 @@ class HeatScenario(base.Scenario): return stack + @base.atomic_action_timer("heat.update_stack") + def _update_stack(self, stack, template=None): + """Update an existing stack + + :param stack: stack that need to be updated + :param template: Updated template + :return: object of updated stack + """ + + template = template or self.default_template + + kw = { + "stack_name": stack.stack_name, + "disable_rollback": True, + "parameters": {}, + "template": template, + "files": {}, + "environment": {} + } + self.clients("heat").stacks.update(stack.id, **kw) + + time.sleep(CONF.benchmark.heat_stack_update_prepoll_delay) + stack = bench_utils.wait_for( + stack, + is_ready=bench_utils.resource_is("UPDATE_COMPLETE"), + update_resource=bench_utils.get_from_manager(["UPDATE_FAILED"]), + timeout=CONF.benchmark.heat_stack_update_timeout, + check_interval=CONF.benchmark.heat_stack_update_poll_interval) + return stack + @base.atomic_action_timer("heat.delete_stack") def _delete_stack(self, stack): """Delete given stack. @@ -108,4 +148,4 @@ class HeatScenario(base.Scenario): stack, update_resource=bench_utils.get_from_manager(), timeout=CONF.benchmark.heat_stack_delete_timeout, - check_interval=CONF.benchmark.heat_stack_delete_poll_interval) + check_interval=CONF.benchmark.heat_stack_delete_poll_interval) \ No newline at end of file diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json new file mode 100644 index 0000000000..c36f759abf --- /dev/null +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json @@ -0,0 +1,21 @@ +{ + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template", + "updated_template_path": "templates/updated_random_strings_add.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ] +} diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml new file mode 100644 index 0000000000..6a66d70a54 --- /dev/null +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml @@ -0,0 +1,14 @@ +--- + HeatStacks.create_update_delete_stack: + - + args: + template_path: "templates/random_strings.yaml.template" + updated_template_path: "templates/updated_random_strings_add.yaml.template" + runner: + type: "constant" + times: 10 + concurrency: 2 + context: + users: + tenants: 2 + users_per_tenant: 3 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json new file mode 100644 index 0000000000..5e460cc16b --- /dev/null +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json @@ -0,0 +1,21 @@ +{ + "HeatStacks.create_update_delete_stack": [ + { + "args": { + "template_path": "templates/random_strings.yaml.template", + "updated_template_path": "templates/updated_random_strings_delete.yaml.template" + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 2 + }, + "context": { + "users": { + "tenants": 2, + "users_per_tenant": 3 + } + } + } + ] +} diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml new file mode 100644 index 0000000000..13e5432ab9 --- /dev/null +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml @@ -0,0 +1,14 @@ +--- + HeatStacks.create_update_delete_stack: + - + args: + template_path: "templates/random_strings.yaml.template" + updated_template_path: "templates/updated_random_strings_delete.yaml.template" + runner: + type: "constant" + times: 10 + concurrency: 2 + context: + users: + tenants: 2 + users_per_tenant: 3 diff --git a/samples/tasks/scenarios/heat/templates/random_strings.yaml.template b/samples/tasks/scenarios/heat/templates/random_strings.yaml.template new file mode 100644 index 0000000000..2dd676c118 --- /dev/null +++ b/samples/tasks/scenarios/heat/templates/random_strings.yaml.template @@ -0,0 +1,13 @@ +heat_template_version: 2014-10-16 + +description: Test template for rally create-update-delete scenario + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_two: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template b/samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template new file mode 100644 index 0000000000..e06d42e012 --- /dev/null +++ b/samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template @@ -0,0 +1,19 @@ +heat_template_version: 2014-10-16 + +description: > + Test template for create-update-delete-stack scenario in rally. + The template updates the stack defined by random_strings.yaml.template with additional resource. + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_two: + type: OS::Heat::RandomString + properties: + length: 20 + test_string_three: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template b/samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template new file mode 100644 index 0000000000..d02593e3b8 --- /dev/null +++ b/samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template @@ -0,0 +1,11 @@ +heat_template_version: 2014-10-16 + +description: > + Test template for create-update-delete-stack scenario in rally. + The template deletes one resource from the stack defined by random_strings.yaml.template. + +resources: + test_string_one: + type: OS::Heat::RandomString + properties: + length: 20 \ No newline at end of file diff --git a/tests/unit/benchmark/scenarios/heat/test_stacks.py b/tests/unit/benchmark/scenarios/heat/test_stacks.py index 56430ca219..25e6fabb98 100644 --- a/tests/unit/benchmark/scenarios/heat/test_stacks.py +++ b/tests/unit/benchmark/scenarios/heat/test_stacks.py @@ -59,5 +59,21 @@ class HeatStacksTestCase(test.TestCase): mock_random_name.return_value = "test-rally-stack" heat_scenario.create_and_delete_stack() - self.assertEqual(1, mock_create.called) + self.assertTrue(mock_create.called) mock_delete.assert_called_once_with(fake_stack) + + @mock.patch(HEAT_STACKS + "._generate_random_name") + @mock.patch(HEAT_STACKS + "._delete_stack") + @mock.patch(HEAT_STACKS + "._update_stack") + @mock.patch(HEAT_STACKS + "._create_stack") + def test_create_update_delete_stack(self, mock_create, mock_update, + mock_delete, mock_random_name): + heat_scenario = stacks.HeatStacks() + fake_stack = object() + mock_create.return_value = fake_stack + mock_random_name.return_value = "test-rally-stack" + heat_scenario.create_update_delete_stack() + + self.assertTrue(mock_create.called) + mock_update.assert_called_once_with(fake_stack, None) + mock_delete.assert_called_once_with(fake_stack) \ No newline at end of file diff --git a/tests/unit/benchmark/scenarios/heat/test_utils.py b/tests/unit/benchmark/scenarios/heat/test_utils.py index 271650d65d..2e7256642a 100644 --- a/tests/unit/benchmark/scenarios/heat/test_utils.py +++ b/tests/unit/benchmark/scenarios/heat/test_utils.py @@ -25,7 +25,6 @@ HEAT_UTILS = "rally.benchmark.scenarios.heat.utils" class HeatScenarioTestCase(test.TestCase): - def setUp(self): super(HeatScenarioTestCase, self).setUp() self.stack = mock.Mock() @@ -59,7 +58,7 @@ class HeatScenarioTestCase(test.TestCase): } mock_clients("heat").stacks.get.return_value = self.stack scenario = utils.HeatScenario() - return_stack = scenario._create_stack("stack_name") + return_stack = scenario._create_stack() self.wait_for.mock.assert_called_once_with(self.stack, update_resource=self.gfm(), is_ready=self.res_is.mock(), @@ -70,6 +69,25 @@ class HeatScenarioTestCase(test.TestCase): self._test_atomic_action_timer(scenario.atomic_actions(), "heat.create_stack") + @mock.patch(HEAT_UTILS + ".HeatScenario.clients") + @mock.patch(HEAT_UTILS + ".CONF.benchmark") + def test_update_stack(self, mock_bench, mock_clients): + mock_clients("heat").stacks.update.return_value = None + mock_bench.heat_stack_update_timeout = 1 + mock_bench.heat_stack_update_poll_interval = 1 + + scenario = utils.HeatScenario() + scenario._update_stack(self.stack, None) + self.wait_for.mock.assert_called_once_with( + self.stack, + update_resource=self.gfm(), + is_ready=self.res_is.mock(), + check_interval=1, + timeout=1) + self.res_is.mock.assert_has_calls([mock.call("UPDATE_COMPLETE")]) + self._test_atomic_action_timer(scenario.atomic_actions(), + "heat.update_stack") + def test_delete_stack(self): scenario = utils.HeatScenario() scenario._delete_stack(self.stack) @@ -84,7 +102,6 @@ class HeatScenarioTestCase(test.TestCase): class HeatScenarioNegativeTestCase(test.TestCase): - @mock.patch(HEAT_UTILS + ".HeatScenario.clients") @mock.patch(HEAT_UTILS + ".CONF.benchmark") def test_failed_create_stack(self, mock_bench, mock_clients): @@ -107,3 +124,23 @@ class HeatScenarioNegativeTestCase(test.TestCase): self.assertIn("has CREATE_FAILED status", str(ex)) except exceptions.TimeoutException: raise self.fail("Unrecognized error status") + + @mock.patch(HEAT_UTILS + ".HeatScenario.clients") + @mock.patch(HEAT_UTILS + ".CONF.benchmark") + def test_failed_update_stack(self, mock_bench, mock_clients): + mock_bench.heat_stack_update_prepoll_delay = 2 + mock_bench.heat_stack_update_timeout = 1 + mock_bench.benchmark.heat_stack_update_poll_interval = 1 + + stack = mock.Mock() + resource = mock.Mock() + resource.stack_status = "UPDATE_FAILED" + stack.manager.get.return_value = resource + mock_clients("heat").stacks.get.return_value = stack + scenario = utils.HeatScenario() + try: + ex = self.assertRaises(exceptions.GetResourceErrorStatus, + scenario._update_stack, stack) + self.assertIn("has UPDATE_FAILED status", str(ex)) + except exceptions.TimeoutException: + raise self.fail("Unrecognized error status") \ No newline at end of file