From f997ca183d9548acc8da41c2bea72db226513954 Mon Sep 17 00:00:00 2001 From: Renat Akhmerov Date: Tue, 29 Jul 2014 12:26:48 +0700 Subject: [PATCH] Implementing DSL specification v2 (partially) * Rewriting package 'workbook' to have v1 and v2 separately * Refactoring all required places to use v1 spec explicitly * Refactoring and making specs more consistent TODO: * Fully implement spec v2 when it's complete Change-Id: I4d6949588635ff45775f68186ea43b16163b6a72 --- mistral/actions/action_factory.py | 13 +- mistral/engine/__init__.py | 8 +- mistral/engine/data_flow.py | 4 +- mistral/exceptions.py | 4 + mistral/services/periodic.py | 7 +- mistral/services/scheduler.py | 9 +- .../controllers/test_workbook_definition.py | 2 +- .../resources/control_flow/direct_flow.yaml | 2 +- .../resources/dsl_v2/reverse_workflow.yaml | 13 ++ mistral/tests/resources/test_rest.yaml | 2 +- .../tests/unit/actions/test_action_factory.py | 15 +- .../unit/actions/test_std_adhoc_action.py | 14 +- .../unit/engine/test_data_flow_module.py | 4 +- mistral/tests/unit/engine/test_task_retry.py | 4 +- mistral/tests/unit/engine/test_workflow.py | 11 +- mistral/tests/unit/utils/test_utils.py | 60 +++++++ .../workbook/v1}/__init__.py | 0 .../test_dsl_specs_v1.py} | 21 ++- .../workbook/{ => v1}/test_get_on_state.py | 8 +- mistral/tests/unit/workbook/v2/__init__.py | 0 .../unit/workbook/v2/test_dsl_specs_v2.py | 166 ++++++++++++++++++ mistral/tests/unit/workflow/__init__.py | 0 .../workflow/test_direct_workflow.py | 0 .../workflow/test_reverse_workflow.py | 21 +++ mistral/utils/__init__.py | 19 +- mistral/workbook/actions.py | 36 ---- mistral/workbook/base.py | 65 +++++-- mistral/workbook/namespaces.py | 69 -------- mistral/workbook/parser.py | 111 ++++++++++++ mistral/workbook/v1/actions.py | 46 +++++ mistral/workbook/v1/namespaces.py | 67 +++++++ mistral/workbook/{ => v1}/tasks.py | 55 +++--- mistral/workbook/{ => v1}/workbook.py | 36 ++-- mistral/workbook/{ => v1}/workflow.py | 19 +- mistral/workbook/v2/actions.py | 68 +++++++ mistral/workbook/v2/namespaces.py | 85 +++++++++ mistral/workbook/v2/retry.py | 49 ++++++ mistral/workbook/v2/tasks.py | 152 ++++++++++++++++ .../v2/triggers.py} | 25 ++- mistral/workbook/v2/workbook.py | 54 ++++++ mistral/workbook/v2/workflows.py | 53 ++++++ mistral/workflow/base.py | 2 - mistral/workflow/direct_workflow.py | 2 - mistral/workflow/reverse_workflow.py | 2 - mistral/workflow/selector.py | 2 - requirements.txt | 1 + 46 files changed, 1162 insertions(+), 244 deletions(-) create mode 100644 mistral/tests/resources/dsl_v2/reverse_workflow.yaml create mode 100644 mistral/tests/unit/utils/test_utils.py rename mistral/tests/{workflow => unit/workbook/v1}/__init__.py (100%) rename mistral/tests/unit/workbook/{test_workbook.py => v1/test_dsl_specs_v1.py} (85%) rename mistral/tests/unit/workbook/{ => v1}/test_get_on_state.py (93%) create mode 100644 mistral/tests/unit/workbook/v2/__init__.py create mode 100644 mistral/tests/unit/workbook/v2/test_dsl_specs_v2.py create mode 100644 mistral/tests/unit/workflow/__init__.py rename mistral/tests/{ => unit}/workflow/test_direct_workflow.py (100%) rename mistral/tests/{ => unit}/workflow/test_reverse_workflow.py (65%) delete mode 100644 mistral/workbook/actions.py delete mode 100644 mistral/workbook/namespaces.py create mode 100644 mistral/workbook/parser.py create mode 100644 mistral/workbook/v1/actions.py create mode 100644 mistral/workbook/v1/namespaces.py rename mistral/workbook/{ => v1}/tasks.py (67%) rename mistral/workbook/{ => v1}/workbook.py (66%) rename mistral/workbook/{ => v1}/workflow.py (67%) create mode 100644 mistral/workbook/v2/actions.py create mode 100644 mistral/workbook/v2/namespaces.py create mode 100644 mistral/workbook/v2/retry.py create mode 100644 mistral/workbook/v2/tasks.py rename mistral/{dsl_parser.py => workbook/v2/triggers.py} (53%) create mode 100644 mistral/workbook/v2/workbook.py create mode 100644 mistral/workbook/v2/workflows.py diff --git a/mistral/actions/action_factory.py b/mistral/actions/action_factory.py index fb6023139..778b38330 100644 --- a/mistral/actions/action_factory.py +++ b/mistral/actions/action_factory.py @@ -22,8 +22,7 @@ from mistral.actions import std_actions from mistral import exceptions as exc from mistral import expressions as expr from mistral.openstack.common import log as logging -from mistral.workbook import actions -from mistral.workbook import tasks +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -112,16 +111,15 @@ def _has_action_context_param(action_cls): def _create_adhoc_action(db_task, openstack_context): - task_spec = tasks.TaskSpec(db_task['task_spec']) + task_spec = spec_parser.get_task_spec(db_task['task_spec']) + full_action_name = task_spec.get_full_action_name() - # TODO(rakhmerov): Fix model attributes during refactoring. raw_action_spec = db_task['action_spec'] - if not raw_action_spec: return None - action_spec = actions.ActionSpec(raw_action_spec) + action_spec = spec_parser.get_action_spec(raw_action_spec) LOG.info('Using ad-hoc action [action=%s, db_task=%s]' % (full_action_name, db_task)) @@ -148,7 +146,8 @@ def _create_adhoc_action(db_task, openstack_context): def create_action(db_task): - task_spec = tasks.TaskSpec(db_task['task_spec']) + task_spec = spec_parser.get_task_spec(db_task['task_spec']) + full_action_name = task_spec.get_full_action_name() action_cls = get_action_class(full_action_name) diff --git a/mistral/engine/__init__.py b/mistral/engine/__init__.py index 5c4e907c9..2432b699f 100644 --- a/mistral/engine/__init__.py +++ b/mistral/engine/__init__.py @@ -29,14 +29,13 @@ cfg.CONF.import_opt('workflow_trace_log_name', 'mistral.config') from mistral.actions import action_factory as a_f from mistral import context as auth_context from mistral.db import api as db_api -from mistral import dsl_parser as parser from mistral.engine import data_flow from mistral.engine import retry from mistral.engine import states from mistral.engine import workflow from mistral import exceptions as exc from mistral.openstack.common import log as logging -from mistral.workbook import tasks as wb_task +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -187,7 +186,7 @@ class Engine(object): WF_TRACE.info("Task '%s' [%s -> %s, result = %s]" % (task.name, task.state, state, result)) - action_name = wb_task.TaskSpec(task.task_spec)\ + action_name = spec_parser.get_task_spec(task.task_spec)\ .get_full_action_name() if not a_f.get_action_class(action_name): @@ -378,7 +377,8 @@ class Engine(object): @classmethod def _get_workbook(cls, workbook_name): wb = db_api.workbook_get(workbook_name) - return parser.get_workbook(wb.definition) + + return spec_parser.get_workbook_spec_from_yaml(wb.definition) @classmethod def _determine_execution_state(cls, execution, tasks): diff --git a/mistral/engine/data_flow.py b/mistral/engine/data_flow.py index 08280ef61..ab1d775ea 100644 --- a/mistral/engine/data_flow.py +++ b/mistral/engine/data_flow.py @@ -24,7 +24,7 @@ from mistral import exceptions as exc from mistral import expressions as expr from mistral.openstack.common import log as logging from mistral.services import trusts -from mistral.workbook import tasks as wb_task +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -87,7 +87,7 @@ def prepare_tasks(tasks_to_start, context, workbook, tasks): # Get action name. Unwrap ad-hoc and reevaluate params if # necessary. - action_name = wb_task.TaskSpec(task.task_spec)\ + action_name = spec_parser.get_task_spec(task.task_spec)\ .get_full_action_name() openstack_ctx = context.get('openstack') diff --git a/mistral/exceptions.py b/mistral/exceptions.py index 48cbbb8c9..df88fbdf4 100644 --- a/mistral/exceptions.py +++ b/mistral/exceptions.py @@ -86,6 +86,10 @@ class ApplicationContextNotFoundException(MistralException): message = "Application context not found" +class DSLParsingException(MistralException): + http_code = 400 + + class InvalidModelException(MistralException): http_code = 400 message = "Wrong entity definition" diff --git a/mistral/services/periodic.py b/mistral/services/periodic.py index a732577df..3bdf8be9d 100644 --- a/mistral/services/periodic.py +++ b/mistral/services/periodic.py @@ -16,13 +16,13 @@ from mistral import context from mistral.db import api as db_api -from mistral import dsl_parser as parser from mistral import engine from mistral.openstack.common import log from mistral.openstack.common import periodic_task from mistral.openstack.common import threadgroup from mistral.services import scheduler as sched from mistral.services import trusts +from mistral.workbook import parser as spec_parser LOG = log.getLogger(__name__) @@ -32,6 +32,7 @@ class MistralPeriodicTasks(periodic_task.PeriodicTasks): def __init__(self, transport=None): super(MistralPeriodicTasks, self).__init__() + self.transport = engine.get_transport(transport) self.engine = engine.EngineClient(self.transport) @@ -44,11 +45,13 @@ class MistralPeriodicTasks(periodic_task.PeriodicTasks): context.set_ctx(ctx) wb = db_api.workbook_get(trigger['workbook_name']) + context.set_ctx(trusts.create_context(wb)) try: - task = parser.get_workbook( + task = spec_parser.get_workbook_spec_from_yaml( wb['definition']).get_trigger_task_name(trigger['name']) + self.engine.start_workflow_execution(wb['name'], task) finally: sched.set_next_execution_time(trigger) diff --git a/mistral/services/scheduler.py b/mistral/services/scheduler.py index 4e3af82ca..d91cecad4 100644 --- a/mistral/services/scheduler.py +++ b/mistral/services/scheduler.py @@ -17,7 +17,7 @@ from croniter import croniter import datetime from mistral.db import api as db_api -from mistral import dsl_parser as parser +from mistral.workbook import parser as spec_parser def get_next_triggers(): @@ -54,8 +54,11 @@ def create_associated_triggers(db_workbook): if not db_workbook['definition']: return - workbook = parser.get_workbook(db_workbook['definition']) - triggers = workbook.get_triggers() + wb_spec = spec_parser.get_workbook_spec_from_yaml( + db_workbook['definition'] + ) + + triggers = wb_spec.get_triggers() # Prepare all triggers data in advance to make db transaction shorter. db_triggers = [] diff --git a/mistral/tests/api/v1/controllers/test_workbook_definition.py b/mistral/tests/api/v1/controllers/test_workbook_definition.py index a34fc98b7..a3c0b7329 100644 --- a/mistral/tests/api/v1/controllers/test_workbook_definition.py +++ b/mistral/tests/api/v1/controllers/test_workbook_definition.py @@ -37,7 +37,7 @@ Workflow: parameters: action: Service:action -triggers: +Triggers: create-vms: type: periodic tasks: create-vms diff --git a/mistral/tests/resources/control_flow/direct_flow.yaml b/mistral/tests/resources/control_flow/direct_flow.yaml index 338a6fea5..80a66f312 100644 --- a/mistral/tests/resources/control_flow/direct_flow.yaml +++ b/mistral/tests/resources/control_flow/direct_flow.yaml @@ -45,7 +45,7 @@ Workflow: task-four: action: MyRest.async-action -triggers: +Triggers: my-cron: type: periodic tasks: start-task diff --git a/mistral/tests/resources/dsl_v2/reverse_workflow.yaml b/mistral/tests/resources/dsl_v2/reverse_workflow.yaml new file mode 100644 index 000000000..4c5fa98d3 --- /dev/null +++ b/mistral/tests/resources/dsl_v2/reverse_workflow.yaml @@ -0,0 +1,13 @@ +--- +Version: '2.0' + +Workflows: + wf1: + type: reverse + tasks: + task1: + action: std.echo output="Hey" + + task2: + action: std.echo output="Hi!" + requires: [task1] diff --git a/mistral/tests/resources/test_rest.yaml b/mistral/tests/resources/test_rest.yaml index a696bad35..13a7fa07f 100644 --- a/mistral/tests/resources/test_rest.yaml +++ b/mistral/tests/resources/test_rest.yaml @@ -93,7 +93,7 @@ Workflow: on-finish: create-vms -triggers: +Triggers: create-vms: type: periodic tasks: create-vms diff --git a/mistral/tests/unit/actions/test_action_factory.py b/mistral/tests/unit/actions/test_action_factory.py index 1124fdd5c..13e8584bd 100644 --- a/mistral/tests/unit/actions/test_action_factory.py +++ b/mistral/tests/unit/actions/test_action_factory.py @@ -20,11 +20,11 @@ import json from mistral.actions import action_factory as a_f from mistral.actions import std_actions as std from mistral.db.sqlalchemy import models -from mistral import dsl_parser as parser from mistral.engine import data_flow from mistral import exceptions from mistral.openstack.common import log as logging from mistral.tests import base +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -323,16 +323,17 @@ class ActionFactoryTest(base.BaseTest): self.assertEqual("'Tango and Cash' is a cool movie!", action.run()) def test_resolve_adhoc_action_name(self): - workbook = parser.get_workbook( + wb = spec_parser.get_workbook_spec_from_yaml( base.get_resource('control_flow/one_sync_task.yaml')) + action_name = 'MyActions.concat' - action = a_f.resolve_adhoc_action_name(workbook, action_name) + action = a_f.resolve_adhoc_action_name(wb, action_name) self.assertEqual('std.echo', action) def test_convert_adhoc_action_params(self): - workbook = parser.get_workbook( + wb = spec_parser.get_workbook_spec_from_yaml( base.get_resource('control_flow/one_sync_task.yaml')) action_name = 'MyActions.concat' @@ -341,20 +342,20 @@ class ActionFactoryTest(base.BaseTest): 'right': 'Stanley' } - parameters = a_f.convert_adhoc_action_params(workbook, + parameters = a_f.convert_adhoc_action_params(wb, action_name, params) self.assertEqual({'output': 'Stormin Stanley'}, parameters) def test_convert_adhoc_action_result(self): - workbook = parser.get_workbook( + wb = spec_parser.get_workbook_spec_from_yaml( base.get_resource('control_flow/one_sync_task.yaml')) action_name = 'MyActions.concat' result = {'output': 'Stormin Stanley'} - parameters = a_f.convert_adhoc_action_result(workbook, + parameters = a_f.convert_adhoc_action_result(wb, action_name, result) diff --git a/mistral/tests/unit/actions/test_std_adhoc_action.py b/mistral/tests/unit/actions/test_std_adhoc_action.py index 2992bb1c5..0e04b5cb2 100644 --- a/mistral/tests/unit/actions/test_std_adhoc_action.py +++ b/mistral/tests/unit/actions/test_std_adhoc_action.py @@ -19,7 +19,7 @@ import copy from mistral.actions import base as actions_base from mistral.actions import std_actions as std from mistral.tests import base -from mistral.workbook import namespaces as ns +from mistral.workbook import parser as spec_parser NS_SPEC = { 'name': 'my_namespace', @@ -67,7 +67,8 @@ class AdHocActionTest(base.BaseTest): def test_adhoc_echo_action(self): ns_raw_spec = copy.copy(NS_SPEC) - action_spec = ns.NamespaceSpec(ns_raw_spec).actions.get('my_action') + action_spec = spec_parser.get_namespace_spec(ns_raw_spec).\ + actions.get('my_action') # With dic-like output formatter. action = std.AdHocAction(None, std.EchoAction, action_spec, @@ -77,7 +78,8 @@ class AdHocActionTest(base.BaseTest): # With list-like output formatter. ns_raw_spec['actions']['my_action']['output'] = ['$', '$'] - action_spec = ns.NamespaceSpec(ns_raw_spec).actions.get('my_action') + action_spec = spec_parser.get_namespace_spec(ns_raw_spec).\ + actions.get('my_action') action = std.AdHocAction(None, std.EchoAction, action_spec, first="Tango", second="Cash") @@ -88,7 +90,8 @@ class AdHocActionTest(base.BaseTest): # With single-object output formatter. ns_raw_spec['actions']['my_action']['output'] = \ "'{$}' is a cool movie!" - action_spec = ns.NamespaceSpec(ns_raw_spec).actions.get('my_action') + action_spec = spec_parser.get_namespace_spec(ns_raw_spec).\ + actions.get('my_action') action = std.AdHocAction(None, std.EchoAction, action_spec, first="Tango", second="Cash") @@ -98,7 +101,8 @@ class AdHocActionTest(base.BaseTest): def test_adhoc_echo_action_with_namespace_parameters(self): ns_raw_spec = copy.copy(NS_SPEC_WITH_PARAMS) - action_spec = ns.NamespaceSpec(ns_raw_spec).actions.get('my_action') + action_spec = spec_parser.get_namespace_spec(ns_raw_spec).\ + actions.get('my_action') action = std.AdHocAction(None, MyAction, action_spec, first="Bifur", diff --git a/mistral/tests/unit/engine/test_data_flow_module.py b/mistral/tests/unit/engine/test_data_flow_module.py index 6ff406986..a2667f527 100644 --- a/mistral/tests/unit/engine/test_data_flow_module.py +++ b/mistral/tests/unit/engine/test_data_flow_module.py @@ -22,7 +22,7 @@ from mistral.engine import data_flow from mistral.engine import states from mistral.openstack.common import log as logging from mistral.tests import base -from mistral.workbook import workbook +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -93,7 +93,7 @@ class DataFlowModuleTest(base.DbTestCase): self.assertEqual('val32', parameters['p2']) def test_prepare_tasks(self): - wb = workbook.WorkbookSpec(WORKBOOK) + wb = spec_parser.get_workbook_spec(WORKBOOK) tasks = [ db_api.task_create(EXEC_ID, TASK.copy()), diff --git a/mistral/tests/unit/engine/test_task_retry.py b/mistral/tests/unit/engine/test_task_retry.py index eec21265d..558ef62a6 100644 --- a/mistral/tests/unit/engine/test_task_retry.py +++ b/mistral/tests/unit/engine/test_task_retry.py @@ -21,13 +21,13 @@ from oslo.config import cfg from mistral.actions import std_actions from mistral.db import api as db_api from mistral.db.sqlalchemy import models as m -from mistral import dsl_parser as parser from mistral import engine from mistral.engine.drivers.default import engine as concrete_engine from mistral.engine import states from mistral import exceptions as exc from mistral.openstack.common import log as logging from mistral.tests import base +from mistral.workbook import parser as spec_parser LOG = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def get_mock_workbook(file, name='my_wb'): def _get_workbook(workbook_name): wb = db_api.workbook_get(workbook_name) - return parser.get_workbook(wb["definition"]) + return spec_parser.get_workbook_spec_from_yaml(wb["definition"]) class FailBeforeSuccessMocker(object): diff --git a/mistral/tests/unit/engine/test_workflow.py b/mistral/tests/unit/engine/test_workflow.py index 1f7c643c3..749a26362 100644 --- a/mistral/tests/unit/engine/test_workflow.py +++ b/mistral/tests/unit/engine/test_workflow.py @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mistral import dsl_parser as parser from mistral.engine import states from mistral.engine import workflow from mistral.tests import base +from mistral.workbook import parser as spec_parser + TASKS = [ { @@ -51,8 +52,10 @@ class WorkflowTest(base.DbTestCase): super(WorkflowTest, self).setUp() def test_find_workflow_tasks(self): + wb_definition = base.get_resource("test_rest.yaml") + tasks = workflow.find_workflow_tasks( - parser.get_workbook(base.get_resource("test_rest.yaml")), + spec_parser.get_workbook_spec_from_yaml(wb_definition), "attach-volumes" ) @@ -62,8 +65,10 @@ class WorkflowTest(base.DbTestCase): self._assert_single_item(tasks, name='attach-volumes') def test_find_workflow_tasks_order(self): + wb_definition = base.get_resource("test_order.yaml") + tasks = workflow.find_workflow_tasks( - parser.get_workbook(base.get_resource("test_order.yaml")), + spec_parser.get_workbook_spec_from_yaml(wb_definition), 'task' ) diff --git a/mistral/tests/unit/utils/test_utils.py b/mistral/tests/unit/utils/test_utils.py new file mode 100644 index 000000000..91191de30 --- /dev/null +++ b/mistral/tests/unit/utils/test_utils.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.tests import base +from mistral import utils + + +class UtilsTest(base.BaseTest): + def setUp(self): + super(UtilsTest, self).setUp() + + def test_merge_dicts(self): + left = { + 'key1': { + 'key11': "val11" + }, + 'key2': 'val2' + } + + right = { + 'key1': { + 'key11': "val111111", + 'key12': "val12", + 'key13': { + 'key131': 'val131' + } + }, + 'key2': 'val2222', + 'key3': 'val3' + } + + utils.merge_dicts(left, right) + + self.assertDictEqual( + { + 'key1': { + 'key11': "val11", + 'key12': "val12", + 'key13': { + 'key131': 'val131' + } + }, + 'key2': 'val2', + 'key3': 'val3' + }, + left + ) diff --git a/mistral/tests/workflow/__init__.py b/mistral/tests/unit/workbook/v1/__init__.py similarity index 100% rename from mistral/tests/workflow/__init__.py rename to mistral/tests/unit/workbook/v1/__init__.py diff --git a/mistral/tests/unit/workbook/test_workbook.py b/mistral/tests/unit/workbook/v1/test_dsl_specs_v1.py similarity index 85% rename from mistral/tests/unit/workbook/test_workbook.py rename to mistral/tests/unit/workbook/v1/test_dsl_specs_v1.py index e15eac3c2..1f1920689 100644 --- a/mistral/tests/unit/workbook/test_workbook.py +++ b/mistral/tests/unit/workbook/v1/test_dsl_specs_v1.py @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mistral import dsl_parser as parser from mistral.tests import base +from mistral.workbook import parser SIMPLE_WORKBOOK = """ @@ -26,20 +26,23 @@ Workflow: """ -class DSLModelTest(base.BaseTest): +class DSLv1ModelTest(base.BaseTest): def setUp(self): - super(DSLModelTest, self).setUp() + super(DSLv1ModelTest, self).setUp() + + # TODO(rakhmerov): Need to have a dedicated resource. self.doc = base.get_resource("test_rest.yaml") def test_load_dsl(self): - wb = parser.get_workbook(self.doc) + wb = parser.get_workbook_spec_from_yaml(self.doc) self.assertEqual(wb.workflow.tasks.items, wb.tasks.items) self.assertEqual(wb.tasks.get("create-vms").name, "create-vms") self.assertEqual(4, len(wb.namespaces.get("MyRest").actions)) def test_tasks(self): - wb = parser.get_workbook(self.doc) + wb = parser.get_workbook_spec_from_yaml(self.doc) + self.assertEqual(len(wb.tasks), 6) attach_volumes = wb.tasks.get("attach-volumes") @@ -67,7 +70,7 @@ class DSLModelTest(base.BaseTest): self.assertEqual(subseq_finish, {"create-vms": ''}) def test_actions(self): - wb = parser.get_workbook(self.doc) + wb = parser.get_workbook_spec_from_yaml(self.doc) actions = wb.namespaces.get("MyRest").actions @@ -92,7 +95,7 @@ class DSLModelTest(base.BaseTest): self.assertEqual("MyRest", action.namespace) def test_namespaces(self): - wb = parser.get_workbook(self.doc) + wb = parser.get_workbook_spec_from_yaml(self.doc) self.assertEqual(len(wb.namespaces), 2) @@ -101,9 +104,9 @@ class DSLModelTest(base.BaseTest): self.assertEqual(1, len(nova_namespace.actions)) def test_workbook_without_namespaces(self): - parser.get_workbook(SIMPLE_WORKBOOK) + parser.get_workbook_spec_from_yaml(SIMPLE_WORKBOOK) def test_triggers(self): - wb = parser.get_workbook(self.doc) + wb = parser.get_workbook_spec_from_yaml(self.doc) self.assertEqual(len(wb.get_triggers()), 1) diff --git a/mistral/tests/unit/workbook/test_get_on_state.py b/mistral/tests/unit/workbook/v1/test_get_on_state.py similarity index 93% rename from mistral/tests/unit/workbook/test_get_on_state.py rename to mistral/tests/unit/workbook/v1/test_get_on_state.py index bf6219419..d9366d314 100644 --- a/mistral/tests/unit/workbook/test_get_on_state.py +++ b/mistral/tests/unit/workbook/v1/test_get_on_state.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO(rakhmerov): Can we just extend dsl_specs_v1.py and remove this one?. + SAMPLE_TASK_SPEC = { 'action': 'MyRest:create-vm', 'name': 'create-vms', @@ -23,27 +25,31 @@ SAMPLE_TASK_SPEC = { } from mistral.tests import base -from mistral.workbook import tasks +from mistral.workbook.v1 import tasks class GetOnStateTest(base.BaseTest): def setUp(self): super(GetOnStateTest, self).setUp() + self.task = tasks.TaskSpec(SAMPLE_TASK_SPEC) def test_state_finish(self): on_finish = self.task.get_on_finish() + self.assertIsInstance(on_finish, dict) self.assertIn("attach-volumes", on_finish) def test_state_error(self): on_error = self.task.get_on_error() + self.assertIsInstance(on_error, dict) self.assertEqual(len(on_error), 2) self.assertIn("task1", on_error) def test_state_success(self): on_success = self.task.get_on_success() + self.assertIsInstance(on_success, dict) self.assertEqual(len(on_success), 3) self.assertIn("task1", on_success) diff --git a/mistral/tests/unit/workbook/v2/__init__.py b/mistral/tests/unit/workbook/v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/tests/unit/workbook/v2/test_dsl_specs_v2.py b/mistral/tests/unit/workbook/v2/test_dsl_specs_v2.py new file mode 100644 index 000000000..307e8bc47 --- /dev/null +++ b/mistral/tests/unit/workbook/v2/test_dsl_specs_v2.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.tests import base +from mistral.workbook import parser as spec_parser + +VALID_WB = """ +--- +Version: '2.0' + +Namespaces: + ns1: + actions: + action1: + class: std.echo + base-parameters: + output: "Hello {$.name}!" + +Workflows: + wf1: + type: reverse + + parameters: + - name + - age + + tasks: + task1: + action: ns1.action1 name="{$.name}" + + task2: + requires: [task1] + action: std.echo output="Thanks {$.name}!" + + wf2: + type: direct + + tasks: + task3: + workflow: wf1 name="John Doe" age=32 +""" + +# TODO(rakhmerov): Add more tests when v2 spec is complete. +# TODO(rakhmerov): Add negative tests. + + +class DSLv2ModelTest(base.BaseTest): + def setUp(self): + super(DSLv2ModelTest, self).setUp() + + def test_valid_workbook_spec(self): + wb_spec = spec_parser.get_workbook_spec_from_yaml(VALID_WB) + + # Workbook. + ns_specs = wb_spec.get_namespaces() + wf_specs = wb_spec.get_workflows() + tr_specs = wb_spec.get_triggers() + + self.assertEqual('2.0', wb_spec.get_version()) + self.assertIsNotNone(ns_specs) + self.assertIsNotNone(wf_specs) + self.assertIsNone(tr_specs) + + # Namespaces. + self.assertEqual(1, len(ns_specs)) + + ns_spec = ns_specs.get('ns1') + + self.assertIsNotNone(ns_spec) + self.assertEqual('2.0', ns_spec.get_version()) + self.assertEqual('ns1', ns_spec.get_name()) + self.assertIsNone(ns_spec.get_class()) + self.assertIsNone(ns_spec.get_base_parameters()) + self.assertIsNone(ns_spec.get_parameters()) + + action_specs = ns_spec.get_actions() + + self.assertEqual(1, len(action_specs)) + + # Actions. + action_spec = action_specs.get('action1') + + self.assertIsNotNone(action_spec) + self.assertEqual('2.0', action_spec.get_version()) + self.assertEqual('action1', action_spec.get_name()) + self.assertEqual('std.echo', action_spec.get_class()) + self.assertDictEqual( + {'output': 'Hello {$.name}!'}, + action_spec.get_base_parameters() + ) + self.assertDictEqual({}, action_spec.get_parameters()) + self.assertIsNone(action_spec.get_output()) + + # Workflows. + + self.assertEqual(2, len(wf_specs)) + + wf1_spec = wf_specs.get('wf1') + + self.assertEqual('2.0', wf1_spec.get_version()) + self.assertEqual('wf1', wf1_spec.get_name()) + self.assertEqual('reverse', wf1_spec.get_type()) + self.assertEqual(2, len(wf1_spec.get_tasks())) + + # Tasks. + + task1_spec = wf1_spec.get_tasks().get('task1') + + self.assertIsNotNone(task1_spec) + self.assertEqual('2.0', task1_spec.get_version()) + self.assertEqual('task1', task1_spec.get_name()) + self.assertEqual('ns1.action1', task1_spec.get_action_name()) + self.assertEqual('action1', task1_spec.get_short_action_name()) + self.assertEqual({'name': '{$.name}'}, task1_spec.get_parameters()) + + task2_spec = wf1_spec.get_tasks().get('task2') + + self.assertIsNotNone(task2_spec) + self.assertEqual('2.0', task2_spec.get_version()) + self.assertEqual('task2', task2_spec.get_name()) + self.assertEqual('std.echo', task2_spec.get_action_name()) + self.assertEqual('echo', task2_spec.get_short_action_name()) + self.assertEqual('std', task2_spec.get_action_namespace()) + self.assertIsNone(task2_spec.get_workflow_name()) + self.assertIsNone(task2_spec.get_short_workflow_name()) + self.assertIsNone(task2_spec.get_workflow_namespace()) + self.assertEqual( + {'output': 'Thanks {$.name}!'}, + task2_spec.get_parameters() + ) + + wf2_spec = wf_specs.get('wf2') + + self.assertEqual('2.0', wf2_spec.get_version()) + self.assertEqual('wf2', wf2_spec.get_name()) + self.assertEqual('direct', wf2_spec.get_type()) + self.assertEqual(1, len(wf2_spec.get_tasks())) + + task3_spec = wf2_spec.get_tasks().get('task3') + + self.assertIsNotNone(task3_spec) + self.assertEqual('2.0', task3_spec.get_version()) + self.assertEqual('task3', task3_spec.get_name()) + self.assertIsNone(task3_spec.get_action_name()) + self.assertIsNone(task3_spec.get_short_action_name()) + self.assertIsNone(task3_spec.get_action_namespace()) + self.assertEqual('wf1', task3_spec.get_workflow_name()) + self.assertEqual('wf1', task3_spec.get_short_workflow_name()) + self.assertIsNone(task3_spec.get_workflow_namespace()) + self.assertEqual( + {'name': 'John Doe', 'age': '32'}, + task3_spec.get_parameters() + ) diff --git a/mistral/tests/unit/workflow/__init__.py b/mistral/tests/unit/workflow/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mistral/tests/workflow/test_direct_workflow.py b/mistral/tests/unit/workflow/test_direct_workflow.py similarity index 100% rename from mistral/tests/workflow/test_direct_workflow.py rename to mistral/tests/unit/workflow/test_direct_workflow.py diff --git a/mistral/tests/workflow/test_reverse_workflow.py b/mistral/tests/unit/workflow/test_reverse_workflow.py similarity index 65% rename from mistral/tests/workflow/test_reverse_workflow.py rename to mistral/tests/unit/workflow/test_reverse_workflow.py index 48b2053c7..c177b7969 100644 --- a/mistral/tests/workflow/test_reverse_workflow.py +++ b/mistral/tests/unit/workflow/test_reverse_workflow.py @@ -14,14 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +from mistral.db.sqlalchemy import models as m from mistral.openstack.common import log as logging from mistral.tests import base +from mistral.workbook import parser as spec_parser +from mistral.workflow import reverse_workflow as r_wf + LOG = logging.getLogger(__name__) class ReverseWorkflowHandlerTest(base.BaseTest): + def setUp(self): + super(ReverseWorkflowHandlerTest, self).setUp() + + wf_spec = spec_parser.get_workbook_spec_from_yaml( + base.get_resource('dsl_v2/reverse_workflow.yaml') + ) + + exec_db = m.WorkflowExecution() + exec_db.update({ + 'id': '1-2-3-4', + 'wf_spec': wf_spec.to_dict() + }) + + self.handler = r_wf.ReverseWorkflowHandler(exec_db) + def test_start_workflow(self): + # self.handler.start_workflow() + # TODO(rakhmerov): Implement. pass diff --git a/mistral/utils/__init__.py b/mistral/utils/__init__.py index c6a37d6d1..ace089619 100644 --- a/mistral/utils/__init__.py +++ b/mistral/utils/__init__.py @@ -65,7 +65,7 @@ def set_thread_local(var_name, val): if val: gl_storage = _get_greenlet_local_storage() if not gl_storage: - gl_storage =\ + gl_storage = \ _th_loc_storage.greenlet_locals[corolocal.get_ident()] = {} gl_storage[var_name] = val @@ -93,3 +93,20 @@ def log_exec(logger, level=logging.INFO): return _logged return _decorator + + +def merge_dicts(left, right): + """Merges two dictionaries. + + Values of right dictionary recursively get merged into left dictionary. + :param left: Left dictionary. + :param right: Right dictionary. + """ + for k, v in right.iteritems(): + if k not in left: + left[k] = v + else: + left_v = left[k] + + if isinstance(left_v, dict) and isinstance(v, dict): + merge_dicts(left_v, v) diff --git a/mistral/workbook/actions.py b/mistral/workbook/actions.py deleted file mode 100644 index d50e4a5c4..000000000 --- a/mistral/workbook/actions.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from mistral.workbook import base - - -class ActionSpec(base.BaseSpec): - _required_keys = ['name', 'class', 'namespace'] - - def __init__(self, action): - super(ActionSpec, self).__init__(action) - - if self.validate(): - self.name = action['name'] - self.clazz = action['class'] - self.namespace = action['namespace'] - self.base_parameters = action.get('base-parameters', {}) - self.parameters = action.get('parameters', {}) - self.output = action.get('output', {}) - - -class ActionSpecList(base.BaseSpecList): - item_class = ActionSpec diff --git a/mistral/workbook/base.py b/mistral/workbook/base.py index 8acc5f25c..064985ee0 100644 --- a/mistral/workbook/base.py +++ b/mistral/workbook/base.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright 2013 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,26 +12,65 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mistral import exceptions +import jsonschema +import six + +from mistral import exceptions as exc class BaseSpec(object): - _required_keys = [] + # See http://json-schema.org + _schema = { + "type": "object", + } + + _version = "1.0" def __init__(self, data): self._data = data + self.validate() + def validate(self): - if not all(k in self._data for k in self._required_keys): - message = ("Wrong model definition for: %s. It should contain" - " required keys: %s" % (self.__class__.__name__, - self._required_keys)) - raise exceptions.InvalidModelException(message) - return True + try: + jsonschema.validate(self._data, self._schema) + except jsonschema.ValidationError as e: + raise exc.InvalidModelException("Invalid DSL: %s" % e) + + def _spec_property(self, prop_name, spec_cls): + prop_val = self._data.get(prop_name) + + return spec_cls(prop_val) if prop_val else None + + def _inject_version(self, prop_names): + for prop_name in prop_names: + prop_data = self._data.get(prop_name) + + if isinstance(prop_data, dict): + prop_data['version'] = self._version + + def _get_as_dict(self, prop_name): + prop_val = self._data.get(prop_name) + + if not prop_val: + return {} + + if isinstance(prop_val, dict): + return prop_val + elif isinstance(prop_val, list): + result = {} + for t in prop_val: + result.update(t if isinstance(t, dict) else {t: ''}) + return result + elif isinstance(prop_val, six.string_types): + return {prop_val: ''} def to_dict(self): return self._data + def get_version(self): + return self._version + class BaseSpecList(object): item_class = None @@ -42,11 +79,9 @@ class BaseSpecList(object): self.items = {} for k, v in data.items(): - v['name'] = k - self.items[k] = self.item_class(v) - - for name in self: - self.get(name).validate() + if k != 'version': + v['name'] = k + self.items[k] = self.item_class(v) def __iter__(self): return iter(self.items) diff --git a/mistral/workbook/namespaces.py b/mistral/workbook/namespaces.py deleted file mode 100644 index 3d924ce1a..000000000 --- a/mistral/workbook/namespaces.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from mistral.workbook import actions -from mistral.workbook import base - - -def merge_dicts(left, right): - for k, v in right.iteritems(): - if k not in left: - left[k] = v - else: - left_v = left[k] - - if isinstance(left_v, dict) and isinstance(v, dict): - merge_dicts(left_v, v) - - -def merge_base_parameters(action, ns_base_parameters): - if not ns_base_parameters: - return - - if 'base-parameters' not in action: - action['base-parameters'] = ns_base_parameters - return - - action_base_parameters = action['base-parameters'] - - merge_dicts(action_base_parameters, ns_base_parameters) - - -class NamespaceSpec(base.BaseSpec): - _required_keys = ['name', 'actions'] - - def __init__(self, namespace): - super(NamespaceSpec, self).__init__(namespace) - - if self.validate(): - self.name = namespace['name'] - self.clazz = namespace.get('class') - self.base_parameters = namespace.get('base-parameters') - self.parameters = namespace.get('parameters') - - for _, action in namespace['actions'].iteritems(): - action['namespace'] = self.name - - if 'class' not in action: - action['class'] = self.clazz - - merge_base_parameters(action, self.base_parameters) - - self.actions = actions.ActionSpecList(namespace['actions']) - - -class NamespaceSpecList(base.BaseSpecList): - item_class = NamespaceSpec diff --git a/mistral/workbook/parser.py b/mistral/workbook/parser.py new file mode 100644 index 000000000..df46e9173 --- /dev/null +++ b/mistral/workbook/parser.py @@ -0,0 +1,111 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +from yaml import error + +from mistral import exceptions as exc +from mistral.workbook.v1 import actions as actions_v1 +from mistral.workbook.v1 import namespaces as ns_v1 +from mistral.workbook.v1 import tasks as tasks_v1 +from mistral.workbook.v1 import workbook as wb_v1 +from mistral.workbook.v1 import workflow as wf_v1 +from mistral.workbook.v2 import actions as actions_v2 +from mistral.workbook.v2 import namespaces as ns_v2 +from mistral.workbook.v2 import tasks as tasks_v2 +from mistral.workbook.v2 import workbook as wb_v2 +from mistral.workbook.v2 import workflows as wf_v2 + +V1_0 = '1.0' +V2_0 = '2.0' + +ALL_VERSIONS = [V1_0, V2_0] + + +def parse_yaml(text): + """Loads a text in YAML format as dictionary object. + + :param text: YAML text. + :return: Parsed YAML document as dictionary. + """ + + try: + return yaml.safe_load(text) + except error.YAMLError as e: + raise RuntimeError("Definition could not be parsed: %s\n" % e) + + +def _get_spec_version(spec_dict): + # If version is not specified it will '1.0' by default. + ver = V1_0 + + if 'Version' in spec_dict: + ver = spec_dict['Version'] + elif 'version' in spec_dict: + ver = spec_dict['version'] + + if ver not in ALL_VERSIONS: + raise exc.DSLParsingException('Unsupported DSL version: %s' % ver) + + return ver + + +# Factory methods to get specifications either from raw YAML formatted text or +# from dictionaries parsed from YAML formatted text. + + +def get_workbook_spec(spec_dict): + if _get_spec_version(spec_dict) == V1_0: + return wb_v1.WorkbookSpec(spec_dict) + else: + return wb_v2.WorkbookSpec(spec_dict) + + +def get_workbook_spec_from_yaml(text): + spec_dict = parse_yaml(text) + + return get_workbook_spec(spec_dict) + + +def get_namespace_spec(spec_dict): + if _get_spec_version(spec_dict) == V1_0: + return ns_v1.NamespaceSpec(spec_dict) + else: + return ns_v2.NamespaceSpec(spec_dict) + + +def get_action_spec(spec_dict): + if _get_spec_version(spec_dict) == V1_0: + return actions_v1.ActionSpec(spec_dict) + else: + return actions_v2.ActionSpec(spec_dict) + + +def get_workflow_spec(spec_dict): + if _get_spec_version(spec_dict) == V1_0: + return wf_v1.WorkflowSpec(spec_dict) + else: + return wf_v2.WorkflowSpec(spec_dict) + + +def get_task_spec(spec_dict): + if _get_spec_version(spec_dict) == V1_0: + return tasks_v1.TaskSpec(spec_dict) + else: + return tasks_v2.TaskSpec(spec_dict) + + +def get_trigger_spec(spec_dict): + # TODO(rakhmerov): Implement. + pass diff --git a/mistral/workbook/v1/actions.py b/mistral/workbook/v1/actions.py new file mode 100644 index 000000000..4420f3048 --- /dev/null +++ b/mistral/workbook/v1/actions.py @@ -0,0 +1,46 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.workbook import base + + +class ActionSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "class": {"type": "string"}, + "namespace": {"type": "string"}, + "base-parameters": {"type": "object"}, + "parameters": {"type": "array"}, + "output": {}, + }, + "required": ["name", "class", "namespace"], + "additionalProperties": False + } + + def __init__(self, data): + super(ActionSpec, self).__init__(data) + + self.name = data['name'] + self.clazz = data['class'] + self.namespace = data['namespace'] + self.base_parameters = data.get('base-parameters', {}) + self.parameters = data.get('parameters', {}) + self.output = data.get('output') + + +class ActionSpecList(base.BaseSpecList): + item_class = ActionSpec diff --git a/mistral/workbook/v1/namespaces.py b/mistral/workbook/v1/namespaces.py new file mode 100644 index 000000000..49c5ff9fb --- /dev/null +++ b/mistral/workbook/v1/namespaces.py @@ -0,0 +1,67 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral import utils +from mistral.workbook import base +from mistral.workbook.v1 import actions + + +def merge_base_parameters(action, ns_base_parameters): + if not ns_base_parameters: + return + + if 'base-parameters' not in action: + action['base-parameters'] = ns_base_parameters + return + + action_base_parameters = action['base-parameters'] + + utils.merge_dicts(action_base_parameters, ns_base_parameters) + + +class NamespaceSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "class": {"type": ["string", "null"]}, + "base-parameters": {"type": ["object", "null"]}, + "actions": {"type": "object"} + }, + "required": ["name", "actions"], + "additionalProperties": False + } + + def __init__(self, data): + super(NamespaceSpec, self).__init__(data) + + self.name = data['name'] + self.clazz = data.get('class') + self.base_parameters = data.get('base-parameters') + self.parameters = data.get('parameters') + + for _, action in data['actions'].iteritems(): + action['namespace'] = self.name + + if 'class' not in action: + action['class'] = self.clazz + + merge_base_parameters(action, self.base_parameters) + + self.actions = actions.ActionSpecList(data['actions']) + + +class NamespaceSpecList(base.BaseSpecList): + item_class = NamespaceSpec diff --git a/mistral/workbook/tasks.py b/mistral/workbook/v1/tasks.py similarity index 67% rename from mistral/workbook/tasks.py rename to mistral/workbook/v1/tasks.py index 38f7a08fe..a8fb7b9d3 100644 --- a/mistral/workbook/tasks.py +++ b/mistral/workbook/v1/tasks.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. +# Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,24 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six - from mistral.workbook import base class TaskSpec(base.BaseSpec): - _required_keys = ['name', 'action'] + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "action": {"type": ["string", "null"]}, + "parameters": {"type": ["object", "null"]}, + "publish": {"type": ["object", "null"]}, + "retry": {"type": ["object", "null"]}, + "requires": {"type": ["object", "string", "array", "null"]}, + "on-finish": {"type": ["string", "array", "null"]}, + "on-success": {"type": ["string", "array", "null"]}, + "on-error": {"type": ["string", "array", "null"]} + }, + "required": ["name", "action"], + "additionalProperties": False + } - def __init__(self, task): - super(TaskSpec, self).__init__(task) + def __init__(self, data): + super(TaskSpec, self).__init__(data) - self._prepare(task) + self._prepare(data) - if self.validate(): - self.requires = task['requires'] - self.action = task['action'] - self.name = task['name'] - self.parameters = task.get('parameters', {}) + self.requires = data.get('requires') + self.action = data['action'] + self.name = data['name'] + self.parameters = data.get('parameters', {}) def _prepare(self, task): if task: @@ -42,22 +53,6 @@ class TaskSpec(base.BaseSpec): elif isinstance(req, dict): task['requires'] = req - def _get_as_dict(self, key): - tasks = self.get_property(key) - - if not tasks: - return {} - - if isinstance(tasks, dict): - return tasks - elif isinstance(tasks, list): - result = {} - for t in tasks: - result.update(t if isinstance(t, dict) else {t: ''}) - return result - elif isinstance(tasks, six.string_types): - return {tasks: ''} - def get_property(self, property_name, default=None): return self._data.get(property_name, default) diff --git a/mistral/workbook/workbook.py b/mistral/workbook/v1/workbook.py similarity index 66% rename from mistral/workbook/workbook.py rename to mistral/workbook/v1/workbook.py index 968ce6bf3..e8bab9fea 100644 --- a/mistral/workbook/workbook.py +++ b/mistral/workbook/v1/workbook.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. +# Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,28 +13,38 @@ # limitations under the License. from mistral.workbook import base -from mistral.workbook import namespaces -from mistral.workbook import workflow +from mistral.workbook.v1 import namespaces +from mistral.workbook.v1 import workflow class WorkbookSpec(base.BaseSpec): - _required_keys = ['Workflow'] + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "Namespaces": {"type": "object"}, + "Workflow": {"type": "object"}, + "Triggers": {"type": "object"} + }, + "required": ["Workflow"], + "additionalProperties": False + } def __init__(self, doc): super(WorkbookSpec, self).__init__(doc) + self.namespaces = {} - if self.validate(): - ns_dict = self._data.get('Namespaces') + ns_dict = self._data.get('Namespaces') - if ns_dict: - self.namespaces = namespaces.NamespaceSpecList(ns_dict) + if ns_dict: + self.namespaces = namespaces.NamespaceSpecList(ns_dict) - self.workflow = workflow.WorkflowSpec(self._data['Workflow']) - self.tasks = self.workflow.tasks + self.workflow = workflow.WorkflowSpec(self._data['Workflow']) + self.tasks = self.workflow.tasks def get_triggers(self): - triggers_from_data = self._data.get("triggers", None) + triggers_from_data = self._data.get("Triggers", None) if not triggers_from_data: return [] @@ -63,5 +71,5 @@ class WorkbookSpec(base.BaseSpec): return self.namespaces.get(namespace_name).actions def get_trigger_task_name(self, trigger_name): - trigger = self._data["triggers"].get(trigger_name) + trigger = self._data["Triggers"].get(trigger_name) return trigger.get('tasks') if trigger else "" diff --git a/mistral/workbook/workflow.py b/mistral/workbook/v1/workflow.py similarity index 67% rename from mistral/workbook/workflow.py rename to mistral/workbook/v1/workflow.py index 8c30461ec..4f48d6400 100644 --- a/mistral/workbook/workflow.py +++ b/mistral/workbook/v1/workflow.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. +# Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,14 +13,21 @@ # limitations under the License. from mistral.workbook import base -from mistral.workbook import tasks +from mistral.workbook.v1 import tasks class WorkflowSpec(base.BaseSpec): - _required_keys = ['tasks'] + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "tasks": {"type": "object"}, + }, + "required": ["tasks"], + "additionalProperties": False + } def __init__(self, workflow): super(WorkflowSpec, self).__init__(workflow) - if self.validate(): - self.tasks = tasks.TaskSpecList(workflow['tasks']) + self.tasks = tasks.TaskSpecList(workflow['tasks']) diff --git a/mistral/workbook/v2/actions.py b/mistral/workbook/v2/actions.py new file mode 100644 index 000000000..4a55883db --- /dev/null +++ b/mistral/workbook/v2/actions.py @@ -0,0 +1,68 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.workbook import base + +# TODO(rakhmerov): In progress. + + +class ActionSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "class": {"type": "string"}, + "namespace": {"type": "string"}, + "base-parameters": {"type": "object"}, + "parameters": {"type": "array"}, + "output": {}, + }, + "required": ["name", "class", "namespace"], + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(ActionSpec, self).__init__(data) + + self._name = data['name'] + self._class = data['class'] + self._namespace = data['namespace'] + self._base_parameters = data.get('base-parameters', {}) + self._parameters = data.get('parameters', {}) + self._output = data.get('output') + + def get_name(self): + return self._name + + def get_class(self): + return self._class + + def get_namespace(self): + return self._namespace + + def get_base_parameters(self): + return self._base_parameters + + def get_parameters(self): + return self._parameters + + def get_output(self): + return self._output + + +class ActionSpecList(base.BaseSpecList): + item_class = ActionSpec diff --git a/mistral/workbook/v2/namespaces.py b/mistral/workbook/v2/namespaces.py new file mode 100644 index 000000000..82c2c9c41 --- /dev/null +++ b/mistral/workbook/v2/namespaces.py @@ -0,0 +1,85 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral import utils +from mistral.workbook import base +from mistral.workbook.v2 import actions + + +# TODO(rakhmerov): It currently duplicates the method in ../v1/namespaces.py +def merge_base_parameters(action, ns_base_parameters): + if not ns_base_parameters: + return + + if 'base-parameters' not in action: + action['base-parameters'] = ns_base_parameters + return + + action_base_parameters = action['base-parameters'] + + utils.merge_dicts(action_base_parameters, ns_base_parameters) + + +class NamespaceSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "class": {"type": ["string", "null"]}, + "base-parameters": {"type": ["object", "null"]}, + "actions": {"type": "object"} + }, + "required": ["name", "actions"], + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(NamespaceSpec, self).__init__(data) + + self._name = data['name'] + self._clazz = data.get('class') + self._base_parameters = data.get('base-parameters') + self._parameters = data.get('parameters') + + for _, action in data['actions'].iteritems(): + action['namespace'] = self._name + + if 'class' not in action: + action['class'] = self._clazz + + merge_base_parameters(action, self._base_parameters) + + self._actions = self._spec_property('actions', actions.ActionSpecList) + + def get_name(self): + return self._name + + def get_class(self): + return self._clazz + + def get_base_parameters(self): + return self._base_parameters + + def get_parameters(self): + return self._parameters + + def get_actions(self): + return self._actions + + +class NamespaceSpecList(base.BaseSpecList): + item_class = NamespaceSpec diff --git a/mistral/workbook/v2/retry.py b/mistral/workbook/v2/retry.py new file mode 100644 index 000000000..51c54b75f --- /dev/null +++ b/mistral/workbook/v2/retry.py @@ -0,0 +1,49 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.workbook import base + +# TODO(rakhmerov): In progress. + + +class RetrySpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "count": {"type": "integer"}, + "break-on": {"type": "string"}, + "delay": {"type": "integer"} + }, + "required": ["count", "delay"], + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(RetrySpec, self).__init__(data) + + self._count = data['count'] + self._break_on = data['break-on'] + self._delay = data['delay'] + + def get_count(self): + return self._count + + def get_break_on(self): + return self._break_on + + def get_delay(self): + return self._delay diff --git a/mistral/workbook/v2/tasks.py b/mistral/workbook/v2/tasks.py new file mode 100644 index 000000000..f7ac0e505 --- /dev/null +++ b/mistral/workbook/v2/tasks.py @@ -0,0 +1,152 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import six + +from mistral import exceptions as exc +from mistral import utils +from mistral.workbook import base +from mistral.workbook.v2 import retry + +# TODO(rakhmerov): In progress. + +CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*") +PARAMS_PTRN = re.compile("([\w]+)=(\".*\"|\'.*'|[\d\.]*)") + + +class TaskSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "action": {"type": ["string", "null"]}, + "workflow": {"type": ["string", "null"]}, + "parameters": {"type": ["object", "null"]}, + "publish": {"type": ["object", "null"]}, + "retry": {"type": ["object", "null"]}, + "requires": {"type": ["string", "array", "null"]}, + "on-finish": {"type": ["string", "object", "array", "null"]}, + "on-success": {"type": ["string", "object", "array", "null"]}, + "on-error": {"type": ["string", "object", "array", "null"]} + }, + "required": ["name"], + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(TaskSpec, self).__init__(data) + + self._name = data['name'] + self._action = data.get('action', None) + self._workflow = data.get('workflow', None) + self._parameters = data.get('parameters', {}) + self._publish = data.get('publish', {}) + self._retry = self._spec_property('retry', retry.RetrySpec) + self._requires = data.get('requires', []) + + self._process_action_and_workflow() + + def _process_action_and_workflow(self): + if self._action and self._workflow: + msg = "Task properties 'action' and 'workflow' can't be" \ + " specified both:" % self._data + raise exc.InvalidModelException(msg) + + if not self._action and not self._workflow: + msg = "One of task properties 'action' or 'workflow' must be" \ + " specified:" % self._data + raise exc.InvalidModelException(msg) + + if self._action: + self._action, params = self._parse_cmd_and_params(self._action) + elif self._workflow: + self._workflow, params = self._parse_cmd_and_params(self._workflow) + + utils.merge_dicts(self._parameters, params) + + def _parse_cmd_and_params(self, cmd_str): + # TODO(rakhmerov): Try to find a way with one expression. + cmd_matcher = CMD_PTRN.search(cmd_str) + + if not cmd_matcher: + msg = "Invalid action/workflow task property: %s" % cmd_str + raise exc.InvalidModelException(msg) + + cmd = cmd_matcher.group() + + params = {} + for k, v in re.findall(PARAMS_PTRN, cmd_str): + params[k] = v.replace('"', '').replace("'", '') + + return cmd, params + + def get_name(self): + return self._name + + def get_action_name(self): + return self._action if self._action else None + + def get_action_namespace(self): + if not self._action: + return None + + arr = self._action.split('.') + + return arr[0] if len(arr) > 1 else None + + def get_short_action_name(self): + return self._action.split('.')[-1] if self._action else None + + def get_workflow_name(self): + return self._workflow + + def get_workflow_namespace(self): + if not self._workflow: + return None + + arr = self._workflow.split('.') + + return arr[0] if len(arr) > 1 else None + + def get_short_workflow_name(self): + return self._workflow.split('.')[-1] if self._workflow else None + + def get_parameters(self): + return self._parameters + + def get_retry(self): + return self._retry + + def get_requires(self): + if isinstance(self._requires, six.string_types): + return [self._requires] + + return self._requires + + def get_on_finish(self): + return self._get_as_dict("on-finish") + + def get_on_success(self): + return self._get_as_dict("on-success") + + def get_on_error(self): + return self._get_as_dict("on-error") + + +class TaskSpecList(base.BaseSpecList): + item_class = TaskSpec diff --git a/mistral/dsl_parser.py b/mistral/workbook/v2/triggers.py similarity index 53% rename from mistral/dsl_parser.py rename to mistral/workbook/v2/triggers.py index eb975736e..7d882ce0e 100644 --- a/mistral/dsl_parser.py +++ b/mistral/workbook/v2/triggers.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2013 - Mirantis, Inc. +# Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,19 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -from yaml import error +from mistral.workbook import base -from mistral.workbook import workbook +# TODO(rakhmerov): In progress. -def parse(workbook_definition): - """Loads a workbook definition in YAML format as dictionary object.""" - try: - return yaml.safe_load(workbook_definition) - except error.YAMLError as exc: - raise RuntimeError("Definition could not be parsed: %s\n" % exc) +class TriggerSpec(base.BaseSpec): + _version = '2.0' + + def __init__(self, data): + super(TriggerSpec, self).__init__(data) + # TODO(rakhmerov): Implement. -def get_workbook(workbook_definition): - return workbook.WorkbookSpec(parse(workbook_definition)) +class TriggerSpecList(base.BaseSpecList): + item_class = TriggerSpec diff --git a/mistral/workbook/v2/workbook.py b/mistral/workbook/v2/workbook.py new file mode 100644 index 000000000..a21b02d57 --- /dev/null +++ b/mistral/workbook/v2/workbook.py @@ -0,0 +1,54 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.workbook import base +from mistral.workbook.v2 import namespaces as ns +from mistral.workbook.v2 import triggers as tr +from mistral.workbook.v2 import workflows as wf + + +class WorkbookSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "Version": {"value": "2.0"}, + "Namespaces": {"type": "object"}, + "Workflows": {"type": "object"}, + "Triggers": {"type": "object"} + }, + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(WorkbookSpec, self).__init__(data) + + self._inject_version(['Namespaces', 'Workflows', 'Triggers']) + + self._namespaces = \ + self._spec_property('Namespaces', ns.NamespaceSpecList) + self._workflows = \ + self._spec_property('Workflows', wf.WorkflowSpecList) + self._triggers = self._spec_property('Triggers', tr.TriggerSpecList) + + def get_namespaces(self): + return self._namespaces + + def get_workflows(self): + return self._workflows + + def get_triggers(self): + return self._triggers diff --git a/mistral/workbook/v2/workflows.py b/mistral/workbook/v2/workflows.py new file mode 100644 index 000000000..93edd9ae8 --- /dev/null +++ b/mistral/workbook/v2/workflows.py @@ -0,0 +1,53 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.workbook import base +from mistral.workbook.v2 import tasks + + +class WorkflowSpec(base.BaseSpec): + # See http://json-schema.org + _schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "type": {"enum": ["reverse", "direct"]}, + "parameters": {"type": ["array", "null"]}, + "tasks": {"type": "object"}, + }, + "required": ["name", "type", "tasks"], + "additionalProperties": False + } + + _version = '2.0' + + def __init__(self, data): + super(WorkflowSpec, self).__init__(data) + + self._name = data['name'] + self._type = data['type'] + self._tasks = self._spec_property('tasks', tasks.TaskSpecList) + + def get_name(self): + return self._name + + def get_type(self): + return self._type + + def get_tasks(self): + return self._tasks + + +class WorkflowSpecList(base.BaseSpecList): + item_class = WorkflowSpec diff --git a/mistral/workflow/base.py b/mistral/workflow/base.py index 3bcd0ed18..b9618e412 100644 --- a/mistral/workflow/base.py +++ b/mistral/workflow/base.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/mistral/workflow/direct_workflow.py b/mistral/workflow/direct_workflow.py index 8b9b823d3..268f3f9cb 100644 --- a/mistral/workflow/direct_workflow.py +++ b/mistral/workflow/direct_workflow.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/mistral/workflow/reverse_workflow.py b/mistral/workflow/reverse_workflow.py index 43d75f5a6..705b57e41 100644 --- a/mistral/workflow/reverse_workflow.py +++ b/mistral/workflow/reverse_workflow.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/mistral/workflow/selector.py b/mistral/workflow/selector.py index 6980d12e7..6602517fb 100644 --- a/mistral/workflow/selector.py +++ b/mistral/workflow/selector.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Copyright 2014 - Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/requirements.txt b/requirements.txt index ec1043ebb..ba10b3465 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ six>=1.7.0 SQLAlchemy>=0.7.8,!=0.9.5,<=0.9.99 stevedore>=0.14 yaql==0.2.1 # This is not in global requirements +jsonschema>=2.0.0,<3.0.0