openstack/rally
Code Issues Proposed changes

Change file loading mechanism for heat tests

Browse Source 
Improve file loading mechanism for heat tests. Previously,
heat tests loaded template files for every tests. "types.set"
allows to exclude redundant file operations from tests.
In addition, the unit tests for heat rally-scenarios were updated
and mocking of CONF.benchmark was excluded from unit-tests.

Change-Id: Ibdace7bf96ff822701ee8505f1dcd232999a6fe8
This commit is contained in:
kairat_kushaev 2015-04-28 17:16:20 +03:00
parent e7f5d79c4d
commit 346a27eeaa
11 changed files with 93 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
rally-jobs/extra/default.yaml.template Normal file
View File

@ -0,0 +1 @@
heat_template_version: 2014-10-16

4
rally-jobs/rally.yaml
View File

@ -571,6 +571,8 @@
HeatStacks.create_and_list_stack:
-
args:
template_path: "/home/jenkins/.rally/extra/default.yaml.template"
runner:
type: "constant"
times: 6
@ -585,6 +587,8 @@
HeatStacks.create_and_delete_stack:
-
args:
template_path: "/home/jenkins/.rally/extra/default.yaml.template"
runner:
type: "constant"
times: 6

72
rally/benchmark/scenarios/heat/stacks.py
View File

@ -15,6 +15,7 @@
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.heat import utils
from rally.benchmark import types
from rally.benchmark import validation
from rally import consts
@ -25,40 +26,26 @@ class HeatStacks(utils.HeatScenario):
RESOURCE_NAME_PREFIX = "rally_stack_"
RESOURCE_NAME_LENGTH = 7
@staticmethod
def _get_template_from_file(template_path):
template = None
if template_path:
try:
with open(template_path, "r") as f:
template = f.read()
except IOError:
raise IOError("Provided path '%(template_path)s' is not valid"
% {"template_path": template_path})
return template
@types.set(template_path=types.FileType)
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario(context={"cleanup": ["heat"]})
def create_and_list_stack(self, template_path=None):
def create_and_list_stack(self, template_path):
"""Add a stack and then list all stacks.
Measure the "heat stack-create" and "heat stack-list" commands
performance.
:param template_path: path to template file. If None or incorrect,
then default empty template will be used.
:param template_path: path to stack template file
"""
template = self._get_template_from_file(template_path)
self._create_stack(template)
self._create_stack(template_path)
self._list_stacks()
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario()
def list_stacks_and_resources(self):
"""List resources from tenant stacks."""
"""List all resources from tenant stacks."""
stacks = self._list_stacks()
with base.AtomicAction(
@ -66,27 +53,27 @@ class HeatStacks(utils.HeatScenario):
for stack in stacks:
self.clients("heat").resources.list(stack.id)
@types.set(template_path=types.FileType)
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario(context={"cleanup": ["heat"]})
def create_and_delete_stack(self, template_path=None):
def create_and_delete_stack(self, template_path):
"""Add and then delete a stack.
Measure the "heat stack-create" and "heat stack-delete" commands
performance.
:param template_path: path to template file. If None or incorrect,
then default empty template will be used.
:param template_path: path to stack template file
"""
template = self._get_template_from_file(template_path)
stack = self._create_stack(template)
stack = self._create_stack(template_path)
self._delete_stack(stack)
@types.set(template_path=types.FileType)
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario(context={"cleanup": ["heat"]})
def create_check_delete_stack(self, template_path=None):
def create_check_delete_stack(self, template_path):
"""Create, check and delete a stack.
Measure the performance of the following commands:
@ -94,44 +81,37 @@ class HeatStacks(utils.HeatScenario):
- heat action-check
- heat stack-delete
:param template_path: path to template file that will be used for
stack create. If None or incorrect, then
default empty template will be used.
:param template_path: path to stack template file
"""
template = self._get_template_from_file(template_path)
stack = self._create_stack(template)
stack = self._create_stack(template_path)
self._check_stack(stack)
self._delete_stack(stack)
@types.set(template_path=types.FileType,
updated_template_path=types.FileType)
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario(context={"cleanup": ["heat"]})
def create_update_delete_stack(self, template_path=None,
updated_template_path=None):
def create_update_delete_stack(self, template_path, updated_template_path):
"""Add, update and then delete a stack.
Measure the "heat stack-create", "heat stack-update"
and "heat stack-delete" commands performance.
:param template_path: path to template file. If None or incorrect,
then default empty template will be used.
:param updated_template_path: path to template file that will be used
for stack update. If None or incorrect,
then default empty template will be
used instead.
:param template_path: path to stack template file
:param updated_template_path: path to updated stack template file
"""
template = self._get_template_from_file(template_path)
stack = self._create_stack(template)
updated_template = self._get_template_from_file(updated_template_path)
self._update_stack(stack, updated_template)
stack = self._create_stack(template_path)
self._update_stack(stack, updated_template_path)
self._delete_stack(stack)
@types.set(template_path=types.FileType)
@validation.required_services(consts.Service.HEAT)
@validation.required_openstack(users=True)
@base.scenario(context={"cleanup": ["heat"]})
def create_suspend_resume_delete_stack(self, template_path=None):
def create_suspend_resume_delete_stack(self, template_path):
"""Create, suspend-resume and then delete a stack.
Measure performance of the following commands:
@ -140,12 +120,10 @@ class HeatStacks(utils.HeatScenario):
heat action-resume
heat stack-delete
:param template_path: path to template file. If None or incorrect,
then default empty template will be used.
:param template_path: path to stack template file
"""
template = self._get_template_from_file(template_path)
s = self._create_stack(template)
s = self._create_stack(template_path)
self._suspend_stack(s)
self._resume_stack(s)
self._delete_stack(s)

11
rally/benchmark/scenarios/heat/utils.py
View File

@ -82,8 +82,6 @@ CONF.register_opts(HEAT_BENCHMARK_OPTS, group=benchmark_group)
class HeatScenario(base.Scenario):
"""Base class for Heat scenarios with basic atomic actions."""
default_template = "HeatTemplateFormatVersion: '2012-12-12'"
@base.atomic_action_timer("heat.list_stacks")
def _list_stacks(self):
"""Return user stack list."""
@ -91,15 +89,14 @@ class HeatScenario(base.Scenario):
return list(self.clients("heat").stacks.list())
@base.atomic_action_timer("heat.create_stack")
def _create_stack(self, template=None):
def _create_stack(self, template):
"""Create a new stack.
:param template: optional parameter. Template with stack description.
:param template: template with stack description.
:returns: object of stack
"""
stack_name = self._generate_random_name()
template = template or self.default_template
kw = {
"stack_name": stack_name,
"disable_rollback": True,
@ -126,7 +123,7 @@ class HeatScenario(base.Scenario):
return stack
@base.atomic_action_timer("heat.update_stack")
def _update_stack(self, stack, template=None):
def _update_stack(self, stack, template):
"""Update an existing stack
:param stack: stack that need to be updated
@ -134,8 +131,6 @@ class HeatScenario(base.Scenario):
:returns: object of updated stack
"""
template = template or self.default_template
kw = {
"stack_name": stack.stack_name,
"disable_rollback": True,

3
samples/tasks/scenarios/heat/create-and-delete-stack.json
View File

@ -1,6 +1,9 @@
{
"HeatStacks.create_and_delete_stack": [
{
"args": {
"template_path": "templates/default.yaml.template"
},
"runner": {
"type": "constant",
"times": 10,

2
samples/tasks/scenarios/heat/create-and-delete-stack.yaml
View File

@ -1,6 +1,8 @@
---
HeatStacks.create_and_delete_stack:
-
args:
template_path: "templates/default.yaml.template"
runner:
type: "constant"
times: 10

3
samples/tasks/scenarios/heat/create-and-list-stack.json
View File

@ -1,6 +1,9 @@
{
"HeatStacks.create_and_list_stack": [
{
"args": {
"template_path": "templates/default.yaml.template"
},
"runner": {
"type": "constant",
"times": 10,

2
samples/tasks/scenarios/heat/create-and-list-stack.yaml
View File

@ -1,6 +1,8 @@
---
HeatStacks.create_and_list_stack:
-
args:
template_path: "templates/default.yaml.template"
runner:
type: "constant"
times: 10

1
samples/tasks/scenarios/heat/templates/default.yaml.template Normal file
View File

@ -0,0 +1 @@
heat_template_version: 2014-10-16

46
tests/unit/benchmark/scenarios/heat/test_stacks.py
View File

@ -13,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import tempfile
import mock
from rally.benchmark.scenarios.heat import stacks
@ -25,16 +23,20 @@ HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks"
class HeatStacksTestCase(test.TestCase):
def setUp(self):
super(HeatStacksTestCase, self).setUp()
self.default_template = "heat_template_version: 2013-05-23"
@mock.patch(HEAT_STACKS + "._generate_random_name")
@mock.patch(HEAT_STACKS + "._list_stacks")
@mock.patch(HEAT_STACKS + "._create_stack")
def test_create_and_list_stack(self, mock_create, mock_list,
mock_random_name):
template_file = tempfile.NamedTemporaryFile()
heat_scenario = stacks.HeatStacks()
mock_random_name.return_value = "test-rally-stack"
heat_scenario.create_and_list_stack(template_path=template_file.name)
self.assertEqual(1, mock_create.called)
heat_scenario.create_and_list_stack(
template_path=self.default_template)
mock_create.assert_called_once_with(self.default_template)
mock_list.assert_called_once_with()
@mock.patch(HEAT_STACKS + ".clients")
@ -59,17 +61,6 @@ class HeatStacksTestCase(test.TestCase):
self._test_atomic_action_timer(
heat_scenario.atomic_actions(), "heat.list_events_of_1_stacks")
@mock.patch(HEAT_STACKS + "._generate_random_name")
@mock.patch(HEAT_STACKS + "._list_stacks")
@mock.patch(HEAT_STACKS + "._create_stack")
def test_create_and_list_stack_fails(self, mock_create, mock_list,
mock_random_name):
heat_scenario = stacks.HeatStacks()
mock_random_name.return_value = "test-rally-stack"
self.assertRaises(IOError,
heat_scenario.create_and_list_stack,
template_path="/tmp/dummy")
@mock.patch(HEAT_STACKS + "._generate_random_name")
@mock.patch(HEAT_STACKS + "._delete_stack")
@mock.patch(HEAT_STACKS + "._create_stack")
@ -79,9 +70,10 @@ class HeatStacksTestCase(test.TestCase):
fake_stack = object()
mock_create.return_value = fake_stack
mock_random_name.return_value = "test-rally-stack"
heat_scenario.create_and_delete_stack()
heat_scenario.create_and_delete_stack(
template_path=self.default_template)
self.assertTrue(mock_create.called)
mock_create.assert_called_once_with(self.default_template)
mock_delete.assert_called_once_with(fake_stack)
@mock.patch(HEAT_STACKS + "._delete_stack")
@ -91,8 +83,9 @@ class HeatStacksTestCase(test.TestCase):
mock_delete):
heat_scenario = stacks.HeatStacks()
mock_create.return_value = "fake_stack_create_check_delete"
heat_scenario.create_check_delete_stack()
mock_create.assert_called_once_with(None)
heat_scenario.create_check_delete_stack(
template_path=self.default_template)
mock_create.assert_called_once_with(self.default_template)
mock_check.assert_called_once_with("fake_stack_create_check_delete")
mock_delete.assert_called_once_with("fake_stack_create_check_delete")
@ -106,10 +99,12 @@ class HeatStacksTestCase(test.TestCase):
fake_stack = object()
mock_create.return_value = fake_stack
mock_random_name.return_value = "test-rally-stack"
heat_scenario.create_update_delete_stack()
heat_scenario.create_update_delete_stack(
template_path=self.default_template,
updated_template_path=self.default_template)
self.assertTrue(mock_create.called)
mock_update.assert_called_once_with(fake_stack, None)
mock_create.assert_called_once_with(self.default_template)
mock_update.assert_called_once_with(fake_stack, self.default_template)
mock_delete.assert_called_once_with(fake_stack)
@mock.patch(HEAT_STACKS + "._delete_stack")
@ -123,9 +118,10 @@ class HeatStacksTestCase(test.TestCase):
mock_delete):
heat_scenario = stacks.HeatStacks()
mock_create.return_value = "fake_stack_create_suspend_resume_delete"
heat_scenario.create_suspend_resume_delete_stack()
heat_scenario.create_suspend_resume_delete_stack(
template_path=self.default_template)
mock_create.assert_called_once_with(None)
mock_create.assert_called_once_with(self.default_template)
mock_suspend.assert_called_once_with(
"fake_stack_create_suspend_resume_delete")
mock_resume.assert_called_once_with(

73
tests/unit/benchmark/scenarios/heat/test_utils.py
View File

@ -23,6 +23,8 @@ from tests.unit import test
BM_UTILS = "rally.benchmark.utils"
HEAT_UTILS = "rally.benchmark.scenarios.heat.utils"
CONF = utils.CONF
class HeatScenarioTestCase(test.TestCase):
def setUp(self):
@ -40,6 +42,7 @@ class HeatScenarioTestCase(test.TestCase):
self.gfm = self.get_fm.mock
self.useFixture(mockpatch.Patch("time.sleep"))
self.scenario = utils.HeatScenario()
self.default_template = "heat_template_version: 2013-05-23"
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
def test_list_stacks(self, mock_clients):
@ -58,32 +61,29 @@ class HeatScenarioTestCase(test.TestCase):
}
mock_clients("heat").stacks.get.return_value = self.stack
scenario = utils.HeatScenario()
return_stack = scenario._create_stack()
self.wait_for.mock.assert_called_once_with(self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=1,
timeout=3600)
return_stack = scenario._create_stack(self.default_template)
self.wait_for.mock.assert_called_once_with(
self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=CONF.benchmark.heat_stack_create_poll_interval,
timeout=CONF.benchmark.heat_stack_create_timeout)
self.res_is.mock.assert_has_calls([mock.call("CREATE_COMPLETE")])
self.assertEqual(self.wait_for.mock(), return_stack)
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.create_stack")
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
@mock.patch(HEAT_UTILS + ".CONF.benchmark")
def test_update_stack(self, mock_bench, mock_clients):
def test_update_stack(self, mock_clients):
mock_clients("heat").stacks.update.return_value = None
mock_bench.heat_stack_update_timeout = 1
mock_bench.heat_stack_update_poll_interval = 1
scenario = utils.HeatScenario()
scenario._update_stack(self.stack, None)
scenario._update_stack(self.stack, self.default_template)
self.wait_for.mock.assert_called_once_with(
self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=1,
timeout=1)
check_interval=CONF.benchmark.heat_stack_update_poll_interval,
timeout=CONF.benchmark.heat_stack_update_timeout)
self.res_is.mock.assert_has_calls([mock.call("UPDATE_COMPLETE")])
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.update_stack")
@ -98,8 +98,8 @@ class HeatScenarioTestCase(test.TestCase):
self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=utils.CONF.benchmark.heat_stack_check_poll_interval,
timeout=utils.CONF.benchmark.heat_stack_check_timeout)
check_interval=CONF.benchmark.heat_stack_check_poll_interval,
timeout=CONF.benchmark.heat_stack_check_timeout)
self.res_is.mock.assert_has_calls([mock.call("CHECK_COMPLETE")])
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.check_stack")
@ -111,17 +111,13 @@ class HeatScenarioTestCase(test.TestCase):
self.wait_for_delete.mock.assert_called_once_with(
self.stack,
update_resource=self.gfm(),
check_interval=1,
timeout=3600)
check_interval=CONF.benchmark.heat_stack_delete_poll_interval,
timeout=CONF.benchmark.heat_stack_delete_timeout)
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.delete_stack")
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
@mock.patch(HEAT_UTILS + ".CONF.benchmark")
def test_suspend_stack(self, mock_bench, mock_clients):
mock_bench.heat_stack_suspend_timeout = 1
mock_bench.heat_stack_suspend_poll_interval = 1
def test_suspend_stack(self, mock_clients):
scenario = utils.HeatScenario()
scenario._suspend_stack(self.stack)
mock_clients("heat").actions.suspend.assert_called_once_with(
@ -130,18 +126,14 @@ class HeatScenarioTestCase(test.TestCase):
self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=1,
timeout=1)
check_interval=CONF.benchmark.heat_stack_suspend_poll_interval,
timeout=CONF.benchmark.heat_stack_suspend_timeout)
self.res_is.mock.assert_has_calls([mock.call("SUSPEND_COMPLETE")])
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.suspend_stack")
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
@mock.patch(HEAT_UTILS + ".CONF.benchmark")
def test_resume_stack(self, mock_bench, mock_clients):
mock_bench.heat_stack_resume_timeout = 1
mock_bench.heat_stack_resume_poll_interval = 1
def test_resume_stack(self, mock_clients):
scenario = utils.HeatScenario()
scenario._resume_stack(self.stack)
mock_clients("heat").actions.resume.assert_called_once_with(
@ -150,8 +142,8 @@ class HeatScenarioTestCase(test.TestCase):
self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=1,
timeout=1)
check_interval=CONF.benchmark.heat_stack_resume_poll_interval,
timeout=CONF.benchmark.heat_stack_resume_timeout)
self.res_is.mock.assert_has_calls([mock.call("RESUME_COMPLETE")])
self._test_atomic_action_timer(scenario.atomic_actions(),
"heat.resume_stack")
@ -159,12 +151,7 @@ class HeatScenarioTestCase(test.TestCase):
class HeatScenarioNegativeTestCase(test.TestCase):
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
@mock.patch(HEAT_UTILS + ".CONF.benchmark")
def test_failed_create_stack(self, mock_bench, mock_clients):
mock_bench.heat_stack_create_prepoll_delay = 2
mock_bench.heat_stack_create_timeout = 1
mock_bench.benchmark.heat_stack_create_poll_interval = 1
def test_failed_create_stack(self, mock_clients):
mock_clients("heat").stacks.create.return_value = {
"stack": {"id": "test_id"}
}
@ -182,12 +169,7 @@ class HeatScenarioNegativeTestCase(test.TestCase):
raise self.fail("Unrecognized error status")
@mock.patch(HEAT_UTILS + ".HeatScenario.clients")
@mock.patch(HEAT_UTILS + ".CONF.benchmark")
def test_failed_update_stack(self, mock_bench, mock_clients):
mock_bench.heat_stack_update_prepoll_delay = 2
mock_bench.heat_stack_update_timeout = 1
mock_bench.benchmark.heat_stack_update_poll_interval = 1
def test_failed_update_stack(self, mock_clients):
stack = mock.Mock()
resource = mock.Mock()
resource.stack_status = "UPDATE_FAILED"
@ -196,7 +178,8 @@ class HeatScenarioNegativeTestCase(test.TestCase):
scenario = utils.HeatScenario()
try:
ex = self.assertRaises(exceptions.GetResourceErrorStatus,
scenario._update_stack, stack)
scenario._update_stack, stack,
"heat_template_version: 2013-05-23")
self.assertIn("has UPDATE_FAILED status", str(ex))
except exceptions.TimeoutException:
raise self.fail("Unrecognized error status")