From 9be4f8e11937eef91365fb15b224e28ba14516e5 Mon Sep 17 00:00:00 2001 From: ali Date: Thu, 1 Oct 2020 10:35:39 +0000 Subject: [PATCH] created a new Api endpoints and added dynamic actions * added dynamic actions: these actions are created and modified in runtime, each action needs a code source to be imported from and a class name. - there are 2 new endpoints: - /v2/code_sources/: used to add new code sources to mistral. - /v2/dynamic_actions/: used to add dynamic actions to mistral in runtime - a new Action provider (DynamicActionProvider) was added: it provides the actions created from the dynamic actions api. Change-Id: I9fe8c28ffdef71016d9dc13aea60a288c8ebaa0a Signed-off-by: ali --- mistral/actions/dynamic_action.py | 316 ++++++++++++++++++ mistral/api/controllers/v2/code_source.py | 222 ++++++++++++ mistral/api/controllers/v2/dynamic_action.py | 227 +++++++++++++ mistral/api/controllers/v2/resources.py | 144 ++++++++ mistral/api/controllers/v2/root.py | 4 + .../versions/040_create_actions_new_tables.py | 80 +++++ mistral/db/v2/api.py | 57 +++- mistral/db/v2/sqlalchemy/api.py | 136 +++++++- mistral/db/v2/sqlalchemy/models.py | 50 +++ mistral/policies/__init__.py | 4 + mistral/policies/code_sources.py | 83 +++++ mistral/policies/dynamic_actions.py | 83 +++++ mistral/services/actions.py | 1 - mistral/services/code_sources.py | 97 ++++++ mistral/services/dynamic_actions.py | 82 +++++ .../actions/test_dynamic_action_provider.py | 126 +++++++ .../tests/unit/api/v2/test_code_sources.py | 164 +++++++++ .../tests/unit/api/v2/test_dynamic_actions.py | 174 ++++++++++ releasenotes/notes/add_dynamic_actions.yaml | 13 + setup.cfg | 1 + 20 files changed, 2060 insertions(+), 4 deletions(-) create mode 100644 mistral/actions/dynamic_action.py create mode 100644 mistral/api/controllers/v2/code_source.py create mode 100644 mistral/api/controllers/v2/dynamic_action.py create mode 100644 mistral/db/sqlalchemy/migration/alembic_migrations/versions/040_create_actions_new_tables.py create mode 100644 mistral/policies/code_sources.py create mode 100644 mistral/policies/dynamic_actions.py create mode 100644 mistral/services/code_sources.py create mode 100644 mistral/services/dynamic_actions.py create mode 100644 mistral/tests/unit/actions/test_dynamic_action_provider.py create mode 100644 mistral/tests/unit/api/v2/test_code_sources.py create mode 100644 mistral/tests/unit/api/v2/test_dynamic_actions.py create mode 100644 releasenotes/notes/add_dynamic_actions.yaml diff --git a/mistral/actions/dynamic_action.py b/mistral/actions/dynamic_action.py new file mode 100644 index 000000000..d320be583 --- /dev/null +++ b/mistral/actions/dynamic_action.py @@ -0,0 +1,316 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import collections +from oslo_config import cfg +import types + +from mistral import exceptions as exc +from mistral_lib import actions as ml_actions +from mistral_lib import serialization +from mistral_lib.utils import inspect_utils + +from mistral.db.v2 import api as db_api +from mistral.services import code_sources as code_sources_service + +CONF = cfg.CONF + + +class DynamicAction(ml_actions.Action): + + def __init__(self, action, code_source_id, namespace=''): + self.action = action + self.namespace = namespace + self.code_source_id = code_source_id + + @classmethod + def get_serialization_key(cls): + return '%s.%s' % (DynamicAction.__module__, DynamicAction.__name__) + + def run(self, context): + return self.action.run(context) + + def is_sync(self): + return self.action.is_sync() + + +class DynamicActionDescriptor(ml_actions.PythonActionDescriptor): + def __init__(self, name, cls_name, action_cls, version, code_source_id, + action_cls_attrs=None, namespace='', project_id=None, + scope=None): + super(DynamicActionDescriptor, self).__init__( + name, + action_cls, + action_cls_attrs, + namespace, + project_id, + scope + ) + + self.cls_name = cls_name + self.version = version + self.code_source_id = code_source_id + + def __repr__(self): + return 'Dynamic action [name=%s, cls=%s , code_source_id=%s,' \ + ' version=%s]' % ( + self.name, + self._action_cls, + self.code_source_id, + self.version + ) + + def instantiate(self, params, wf_ctx): + if not self._action_cls_attrs: + # No need to create new dynamic type. + return DynamicAction( + self._action_cls(**params), + self.code_source_id, + self.namespace) + + dynamic_cls = type( + self._action_cls.__name__, + (self._action_cls,), + **self._action_cls_attrs + ) + + return DynamicAction( + dynamic_cls(**params), + self.code_source_id, + self.namespace + ) + + +class DynamicActionSerializer(serialization.DictBasedSerializer): + def serialize_to_dict(self, entity): + cls = type(entity.action) + + return { + 'cls_name': cls.__name__, + 'cls_attrs': inspect_utils.get_public_fields(cls), + 'data': vars(entity.action), + 'code_source_id': entity.code_source_id, + 'namespace': entity.namespace, + } + + def deserialize_from_dict(self, entity_dict): + cls_name = entity_dict['cls_name'] + + mod = _get_module( + entity_dict['code_source_id'], + entity_dict['namespace'] + ) + + cls = getattr(mod[0], cls_name) + + cls_attrs = entity_dict['cls_attrs'] + + if cls_attrs: + cls = type(cls.__name__, (cls,), cls_attrs) + + action = cls.__new__(cls) + + for k, v in entity_dict['data'].items(): + setattr(action, k, v) + + return DynamicAction( + action, + entity_dict['code_source_id'], + entity_dict['namespace'] + ) + + +def _get_module(code_source_id, namespace=''): + code_source = code_sources_service.get_code_source( + code_source_id, + namespace + ) + + mod = _load_module(code_source.name, code_source.src) + + return mod, code_source.version + + +def _load_module(fullname, content): + mod = types.ModuleType(fullname) + + exec(content, mod.__dict__) + + return mod + + +serialization.register_serializer(DynamicAction, DynamicActionSerializer()) + + +class DynamicActionProvider(ml_actions.ActionProvider): + """Provides dynamic actions.""" + + def __init__(self, name='dynamic'): + super().__init__(name) + + self._action_descs = collections.OrderedDict() + self._code_sources = collections.OrderedDict() + + def _get_code_source_version(self, code_src_id, namespace=''): + code_src = code_sources_service.get_code_source( + code_src_id, + namespace, + fields=['version'] + ) + + return code_src[0] + + def _load_code_source(self, id): + mod_pair = _get_module(id) + self._code_sources[id] = mod_pair + + return mod_pair + + def _get_code_source(self, id): + mod_pair = self._code_sources.get(id) + code_src_db_version = self._get_code_source_version(id) + + if not mod_pair or mod_pair[1] != code_src_db_version: + mod_pair = self._load_code_source(id) + + return mod_pair + + def _get_action_from_db(self, name, namespace, fields=()): + action = None + + try: + action = db_api.get_dynamic_action( + identifier=name, + namespace=namespace, + fields=fields + ) + except exc.DBEntityNotFoundError: + pass + + return action + + def _action_exists_in_db(self, name, namespace): + action = self._get_action_from_db( + name, + namespace, + fields=['name'] + ) + + return action is not None + + def _reload_action(self, action_desc, mod_pair): + action_desc._action_cls = getattr( + mod_pair[0], + action_desc.cls_name + ) + + action_desc.version = mod_pair[1] + + def _load_new_action(self, action_name, namespace, action_def): + # only query the db if action_def was None + action_def = action_def or self._get_action_from_db( + action_name, + namespace=namespace + ) + + if not action_def: + return + + mod_pair = self._get_code_source(action_def.code_source_id) + + cls = getattr(mod_pair[0], action_def.class_name) + + action_desc = DynamicActionDescriptor( + name=action_def.name, + action_cls=cls, + cls_name=action_def.class_name, + version=1, + code_source_id=action_def.code_source_id + ) + + self._action_descs[(action_name, namespace)] = action_desc + + return action_desc + + def _load_existing_action(self, action_desc, action_name, namespace): + if not self._action_exists_in_db(action_name, namespace=namespace): + # deleting action from cache + del self._action_descs[(action_name, namespace)] + + return + + mod_pair = self._get_code_source(action_desc.code_source_id) + + if action_desc.version != mod_pair[1]: + self._reload_action(action_desc, mod_pair) + + return action_desc + + def _load_action(self, action_name, namespace=None, action_def=None): + action_desc = self._action_descs.get((action_name, namespace)) + + if action_desc: + action_desc = self._load_existing_action( + action_desc, + action_name, + namespace + ) + else: + action_desc = self._load_new_action( + action_name, + namespace, + action_def + ) + + return action_desc + + def find(self, action_name, namespace=None): + + return self._load_action(action_name, namespace) + + def _clean_deleted_actions_from_cache(self): + to_delete = [ + key for key in self._action_descs.keys() + if not self._action_exists_in_db(*key) + ] + + for key in to_delete: + del self._action_descs[key] + + def find_all(self, namespace='', limit=None, sort_fields=None, + sort_dirs=None, **filters): + filters = { + 'namespace': {'eq': namespace} + } + self._clean_deleted_actions_from_cache() + + actions = db_api.get_dynamic_actions( + limit=limit, + sort_keys=sort_fields, + sort_dirs=sort_dirs, + **filters + ) + + for action in actions: + self._load_action( + action.name, + namespace=namespace, + action_def=action + ) + + return dict(filter( + lambda elem: elem[0][1] == namespace, + self._action_descs.items()) + ) diff --git a/mistral/api/controllers/v2/code_source.py b/mistral/api/controllers/v2/code_source.py new file mode 100644 index 000000000..9352bb076 --- /dev/null +++ b/mistral/api/controllers/v2/code_source.py @@ -0,0 +1,222 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log as logging +import pecan +from pecan import hooks +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api import access_control as acl +from mistral.api.controllers.v2 import resources +from mistral.api.controllers.v2 import types + +from mistral.api.hooks import content_type as ct_hook +from mistral import context + +from mistral.db.v2 import api as db_api +from mistral.services import code_sources + +from mistral.utils import filter_utils +from mistral.utils import rest_utils + +LOG = logging.getLogger(__name__) + + +class CodeSourcesController(rest.RestController, hooks.HookController): + __hooks__ = [ct_hook.ContentTypeHook("application/json", ['POST', 'PUT'])] + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="multipart/form-data") + def post(self, namespace='', **files): + """Creates new Code Sources. + + :param namespace: Optional. The namespace to create the code sources + in. + :params **files: a list of files to create code sources from, + the variable name of the file will be the module name + """ + + acl.enforce('code_sources:create', context.ctx()) + + LOG.debug( + 'Creating Code Sources with names: %s in namespace:[%s]', + files.keys(), + namespace + ) + + code_sources_db = code_sources.create_code_sources(namespace, **files) + + code_sources_list = [ + resources.CodeSource.from_db_model(db_cs) + for db_cs in code_sources_db + ] + + return resources.CodeSources(code_sources=code_sources_list).to_json() + + @wsme_pecan.wsexpose(resources.CodeSources, types.uuid, int, + types.uniquelist, types.list, types.uniquelist, + wtypes.text, wtypes.text, + resources.SCOPE_TYPES, types.uuid, wtypes.text, + wtypes.text, bool, wtypes.text) + def get_all(self, marker=None, limit=None, sort_keys='created_at', + sort_dirs='asc', fields='', name=None, + tags=None, scope=None, + project_id=None, created_at=None, updated_at=None, + all_projects=False, namespace=None): + """Return a list of Code Sources. + + :param marker: Optional. Pagination marker for large data sets. + :param limit: Optional. Maximum number of resources to return in a + single result. Default value is None for backward + compatibility. + :param sort_keys: Optional. Columns to sort results by. + Default: created_at. + :param sort_dirs: Optional. Directions to sort corresponding to + sort_keys, "asc" or "desc" can be chosen. + Default: asc. + :param fields: Optional. A specified list of fields of the resource to + be returned. 'id' will be included automatically in + fields if it's provided, since it will be used when + constructing 'next' link. + :param name: Optional. Keep only resources with a specific name. + :param namespace: Optional. Keep only resources with a specific + namespace + :param input: Optional. Keep only resources with a specific input. + :param definition: Optional. Keep only resources with a specific + definition. + :param tags: Optional. Keep only resources containing specific tags. + :param scope: Optional. Keep only resources with a specific scope. + :param project_id: Optional. The same as the requester project_id + or different if the scope is public. + :param created_at: Optional. Keep only resources created at a specific + time and date. + :param updated_at: Optional. Keep only resources with specific latest + update time and date. + :param all_projects: Optional. Get resources of all projects. + """ + + acl.enforce('code_sources:list', context.ctx()) + + filters = filter_utils.create_filters_from_request_params( + created_at=created_at, + name=name, + scope=scope, + tags=tags, + updated_at=updated_at, + project_id=project_id, + namespace=namespace + ) + + LOG.debug( + "Fetch code sources. marker=%s, limit=%s, sort_keys=%s, " + "sort_dirs=%s, fields=%s, filters=%s, all_projects=%s", + marker, + limit, + sort_keys, + sort_dirs, + fields, + filters, + all_projects + ) + + return rest_utils.get_all( + resources.CodeSources, + resources.CodeSource, + db_api.get_code_sources, + db_api.get_code_source, + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + fields=fields, + all_projects=all_projects, + **filters + ) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(resources.CodeSource, wtypes.text, wtypes.text) + def get(self, identifier, namespace=''): + """Return the named Code Source. + + :param identifier: Name or UUID of the code source to retrieve. + :param namespace: Optional. Namespace of the code source to retrieve. + """ + + acl.enforce('code_sources:get', context.ctx()) + + LOG.debug( + 'Fetch Code Source [identifier=%s], [namespace=%s]', + identifier, + namespace + ) + + db_model = rest_utils.rest_retry_on_db_error( + code_sources.get_code_source)( + identifier=identifier, + namespace=namespace + ) + + return resources.CodeSource.from_db_model(db_model) + + @rest_utils.wrap_pecan_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, wtypes.text, status_code=204) + def delete(self, identifier, namespace=''): + """Delete a Code Source. + + :param identifier: Name or ID of Code Source to delete. + :param namespace: Optional. Namespace of the Code Source to delete. + """ + + acl.enforce('code_sources:delete', context.ctx()) + + LOG.debug( + 'Delete Code Source [identifier=%s, namespace=%s]', + identifier, + namespace + ) + + rest_utils.rest_retry_on_db_error( + code_sources.delete_code_source + )( + identifier=identifier, + namespace=namespace + ) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="multipart/form-data") + def put(self, namespace='', **files): + """Update Code Sources. + + :param namespace: Optional. The namespace of the code sources. + :params **files: a list of files to update code sources from. + """ + acl.enforce('code_sources:update', context.ctx()) + + LOG.debug( + 'Updating Code Sources with names: %s in namespace:[%s]', + files.keys(), + namespace + ) + + code_sources_db = code_sources.update_code_sources(namespace, **files) + + code_sources_list = [ + resources.CodeSource.from_db_model(db_cs) + for db_cs in code_sources_db + ] + + return resources.CodeSources(code_sources=code_sources_list).to_json() diff --git a/mistral/api/controllers/v2/dynamic_action.py b/mistral/api/controllers/v2/dynamic_action.py new file mode 100644 index 000000000..5cfd4767d --- /dev/null +++ b/mistral/api/controllers/v2/dynamic_action.py @@ -0,0 +1,227 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log as logging +import pecan +from pecan import hooks +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api import access_control as acl +from mistral.api.controllers.v2 import resources +from mistral.api.controllers.v2 import types +from mistral.api.hooks import content_type as ct_hook +from mistral import context +from mistral.utils import safe_yaml + +from mistral.db.v2 import api as db_api +from mistral.services import dynamic_actions + +from mistral.utils import filter_utils +from mistral.utils import rest_utils + +LOG = logging.getLogger(__name__) + + +class DynamicActionsController(rest.RestController, hooks.HookController): + __hooks__ = [ct_hook.ContentTypeHook("application/json", ['POST', 'PUT'])] + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def post(self, namespace=''): + """Creates new Actions. + + :param namespace: Optional. The namespace to create the actions in. + + The text is allowed to have multiple Actions, In such case, + they all will be created. + """ + acl.enforce('dynamic_actions:create', context.ctx()) + + actions = safe_yaml.load(pecan.request.text) + + LOG.debug( + 'Creating Actions with names: %s in namespace:[%s]', + actions, + namespace + ) + + actions_db = dynamic_actions.create_dynamic_actions(actions, namespace) + + actions_list = [ + resources.DynamicAction.from_db_model(action) + for action in actions_db + ] + + return resources.DynamicActions( + dynamic_actions=actions_list).to_json() + + @wsme_pecan.wsexpose(resources.DynamicActions, types.uuid, int, + types.uniquelist, types.list, types.uniquelist, + wtypes.text, wtypes.text, + resources.SCOPE_TYPES, types.uuid, wtypes.text, + wtypes.text, bool, wtypes.text) + def get_all(self, marker=None, limit=None, sort_keys='created_at', + sort_dirs='asc', fields='', name=None, + tags=None, scope=None, + project_id=None, created_at=None, updated_at=None, + all_projects=False, namespace=None): + """Return a list of Actions. + + :param marker: Optional. Pagination marker for large data sets. + :param limit: Optional. Maximum number of resources to return in a + single result. Default value is None for backward + compatibility. + :param sort_keys: Optional. Columns to sort results by. + Default: created_at. + :param sort_dirs: Optional. Directions to sort corresponding to + sort_keys, "asc" or "desc" can be chosen. + Default: asc. + :param fields: Optional. A specified list of fields of the resource to + be returned. 'id' will be included automatically in + fields if it's provided, since it will be used when + constructing 'next' link. + :param name: Optional. Keep only resources with a specific name. + :param namespace: Optional. Keep only resources with a specific + namespace + :param input: Optional. Keep only resources with a specific input. + :param definition: Optional. Keep only resources with a specific + definition. + :param tags: Optional. Keep only resources containing specific tags. + :param scope: Optional. Keep only resources with a specific scope. + :param project_id: Optional. The same as the requester project_id + or different if the scope is public. + :param created_at: Optional. Keep only resources created at a specific + time and date. + :param updated_at: Optional. Keep only resources with specific latest + update time and date. + :param all_projects: Optional. Get resources of all projects. + """ + + acl.enforce('dynamic_actions:list', context.ctx()) + + filters = filter_utils.create_filters_from_request_params( + created_at=created_at, + name=name, + scope=scope, + tags=tags, + updated_at=updated_at, + project_id=project_id, + namespace=namespace + ) + + LOG.debug( + "Fetch dynamic actions. marker=%s, limit=%s, sort_keys=%s, " + "sort_dirs=%s, fields=%s, filters=%s, all_projects=%s", + marker, + limit, + sort_keys, + sort_dirs, + fields, + filters, + all_projects + ) + + return rest_utils.get_all( + resources.DynamicActions, + resources.DynamicAction, + db_api.get_dynamic_actions, + db_api.get_dynamic_action, + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + fields=fields, + all_projects=all_projects, + **filters + ) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(resources.DynamicAction, wtypes.text, wtypes.text) + def get(self, identifier, namespace=''): + """Return the named action. + + :param identifier: Name or UUID of the action to retrieve. + :param namespace: Optional. Namespace of the action to retrieve. + """ + acl.enforce('dynamic_actions:get', context.ctx()) + + LOG.debug( + 'Fetch Action [identifier=%s], [namespace=%s]', + identifier, + namespace + ) + + db_model = rest_utils.rest_retry_on_db_error( + dynamic_actions.get_dynamic_action)( + identifier=identifier, + namespace=namespace + ) + + return resources.DynamicAction.from_db_model(db_model) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="multipart/form-data") + def delete(self, identifier, namespace=''): + """Delete an Action. + + :param identifier: Name or ID of Action to delete. + :param namespace: Optional. Namespace of the Action to delete. + """ + + acl.enforce('dynamic_actions:delete', context.ctx()) + + LOG.debug( + 'Delete Action [identifier=%s, namespace=%s]', + identifier, + namespace + ) + + rest_utils.rest_retry_on_db_error( + dynamic_actions.delete_dynamic_action)( + identifier=identifier, + namespace=namespace + ) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def put(self, namespace=''): + """Update Actions. + + :param namespace: Optional. The namespace to update the actions in. + + The text is allowed to have multiple Actions, In such case, + they all will be updated. + """ + acl.enforce('dynamic_actions:update', context.ctx()) + + actions = safe_yaml.load(pecan.request.text) + + LOG.debug( + 'Updating Actions with names: %s in namespace:[%s]', + actions.keys(), + namespace + ) + + actions_db = dynamic_actions.update_dynamic_actions(actions, namespace) + + actions_list = [ + resources.DynamicAction.from_db_model(action) + for action in actions_db + ] + + return resources.DynamicActions( + dynamic_actions=actions_list).to_json() diff --git a/mistral/api/controllers/v2/resources.py b/mistral/api/controllers/v2/resources.py index 90c34a82f..957bce15d 100644 --- a/mistral/api/controllers/v2/resources.py +++ b/mistral/api/controllers/v2/resources.py @@ -185,6 +185,150 @@ class Workflow(resource.Resource, ScopedResource): return obj +class CodeSource(resource.Resource, ScopedResource): + """CodeSource resource.""" + + id = wtypes.text + name = wtypes.text + src = wtypes.text + scope = SCOPE_TYPES + version = wtypes.IntegerType(minimum=1) + + project_id = wsme.wsattr(wtypes.text, readonly=True) + + actions = [wtypes.text] + created_at = wtypes.text + updated_at = wtypes.text + namespace = wtypes.text + + @classmethod + def sample(cls): + return cls( + id='123e4567-e89b-12d3-a456-426655440000', + name='module', + src='content of file', + version=1, + scope='private', + actions=['action1', 'action2', 'action3'], + project_id='a7eb669e9819420ea4bd1453e672c0a7', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000', + namespace='' + ) + + @classmethod + def from_db_model(cls, db_model): + return CodeSource( + id=getattr(db_model, 'id', db_model.name), + name=db_model.name, + version=db_model.version, + src=db_model.src, + namespace=db_model.namespace, + project_id=db_model.project_id, + scope=db_model.scope, + created_at=utils.datetime_to_str( + getattr(db_model, 'created_at', '') + ), + updated_at=utils.datetime_to_str( + getattr(db_model, 'updated_at', '') + ) + ) + + +class CodeSources(resource.ResourceList): + """A collection of CodeSources.""" + + code_sources = [CodeSource] + + def __init__(self, **kwargs): + self._type = 'code_sources' + + super(CodeSources, self).__init__(**kwargs) + + @classmethod + def sample(cls): + code_Source_sample = cls() + code_Source_sample.code_sources = [CodeSource.sample()] + code_Source_sample.next = ( + "http://localhost:8989/v2/code_sources?" + "sort_keys=id,name&" + "sort_dirs=asc,desc&limit=10&" + "marker=123e4567-e89b-12d3-a456-426655440000" + ) + + return code_Source_sample + + +class DynamicAction(resource.Resource, ScopedResource): + """DynamicAction resource.""" + + id = wtypes.text + name = wtypes.text + code_source_id = wtypes.text + class_name = wtypes.text + project_id = wsme.wsattr(wtypes.text, readonly=True) + + created_at = wtypes.text + updated_at = wtypes.text + namespace = wtypes.text + + @classmethod + def sample(cls): + return cls( + id='123e4567-e89b-12d3-a456-426655440000', + name='actionName', + class_name='className', + code_source_id='233e4567-354b-12d3-4444-426655444444', + scope='private', + project_id='a7eb669e9819420ea4bd1453e672c0a7', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000', + namespace='' + ) + + @classmethod + def from_db_model(cls, db_model): + return DynamicAction( + id=getattr(db_model, 'id', db_model.name), + name=db_model.name, + code_source_id=db_model.code_source_id, + class_name=db_model.class_name, + namespace=db_model.namespace, + project_id=db_model.project_id, + scope=db_model.scope, + created_at=utils.datetime_to_str( + getattr(db_model, 'created_at', '') + ), + updated_at=utils.datetime_to_str( + getattr(db_model, 'updated_at', '') + ) + ) + + +class DynamicActions(resource.ResourceList): + """A collection of DynamicActions.""" + + dynamic_actions = [DynamicAction] + + def __init__(self, **kwargs): + self._type = 'dynamic_actions' + + super(DynamicActions, self).__init__(**kwargs) + + @classmethod + def sample(cls): + dynamic_action_sample = cls() + dynamic_action_sample.dynamic_actions = [DynamicAction.sample()] + dynamic_action_sample.next = ( + "http://localhost:8989/v2/dynamic_actions?" + "sort_keys=id,name&" + "sort_dirs=asc,desc&limit=10&" + "marker=123e4567-e89b-12d3-a456-426655440000" + ) + + return dynamic_action_sample + + class Workflows(resource.ResourceList): """A collection of workflows.""" diff --git a/mistral/api/controllers/v2/root.py b/mistral/api/controllers/v2/root.py index 156659f1a..e0ab62128 100644 --- a/mistral/api/controllers/v2/root.py +++ b/mistral/api/controllers/v2/root.py @@ -20,7 +20,9 @@ import wsmeext.pecan as wsme_pecan from mistral.api.controllers import resource from mistral.api.controllers.v2 import action from mistral.api.controllers.v2 import action_execution +from mistral.api.controllers.v2 import code_source from mistral.api.controllers.v2 import cron_trigger +from mistral.api.controllers.v2 import dynamic_action from mistral.api.controllers.v2 import environment from mistral.api.controllers.v2 import event_trigger from mistral.api.controllers.v2 import execution @@ -48,6 +50,8 @@ class Controller(object): workbooks = workbook.WorkbooksController() actions = action.ActionsController() + code_sources = code_source.CodeSourcesController() + dynamic_actions = dynamic_action.DynamicActionsController() workflows = workflow.WorkflowsController() executions = execution.ExecutionsController() tasks = task.TasksController() diff --git a/mistral/db/sqlalchemy/migration/alembic_migrations/versions/040_create_actions_new_tables.py b/mistral/db/sqlalchemy/migration/alembic_migrations/versions/040_create_actions_new_tables.py new file mode 100644 index 000000000..491f2a10e --- /dev/null +++ b/mistral/db/sqlalchemy/migration/alembic_migrations/versions/040_create_actions_new_tables.py @@ -0,0 +1,80 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""create new tables for the dynamic actions and code sources + +Revision ID: 001 +Revises: None +Create Date: 2020-09-30 12:02:51.935368 + +""" + +# revision identifiers, used by Alembic. +revision = '040' +down_revision = '039' + +from alembic import op +from mistral.db.sqlalchemy import types as st +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'code_sources', + + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('project_id', sa.String(length=80), nullable=True), + sa.Column('namespace', sa.String(length=255), nullable=True), + sa.Column('src', sa.TEXT, nullable=False), + sa.Column('version', sa.Integer, nullable=False), + sa.Column('tags', st.JsonEncoded(), nullable=True), + sa.Column('scope', sa.String(length=80), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name', 'namespace', 'project_id'), + + sa.Index('code_sources_project_id', 'project_id'), + sa.Index('code_sources_scope', 'scope') + ) + + op.create_table( + 'dynamic_actions', + + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('class_name', sa.String(length=255), nullable=False), + sa.Column('scope', sa.String(length=80), nullable=True), + sa.Column('project_id', sa.String(length=80), nullable=True), + sa.Column('code_source_id', sa.String(length=36), nullable=False), + sa.Column('namespace', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint( + ['code_source_id'], + [u'code_sources.id'], + ondelete='CASCADE' + ), + + sa.UniqueConstraint('name', 'namespace', 'project_id'), + + sa.Index('dynamic_actions_project_id', 'project_id'), + sa.Index('dynamic_actions_scope', 'scope'), + + ) diff --git a/mistral/db/v2/api.py b/mistral/db/v2/api.py index 69f5df972..23927d67a 100644 --- a/mistral/db/v2/api.py +++ b/mistral/db/v2/api.py @@ -19,7 +19,6 @@ import contextlib from oslo_config import cfg from oslo_db import api as db_api - _BACKEND_MAPPING = { 'sqlalchemy': 'mistral.db.v2.sqlalchemy.api', } @@ -171,7 +170,61 @@ def delete_workflow_definitions(**kwargs): IMPL.delete_workflow_definitions(**kwargs) -# Action definitions. +def create_dynamic_action(values): + return IMPL.create_dynamic_action(values) + + +def delete_dynamic_action(identifier, namespace=''): + return IMPL.delete_dynamic_action(identifier, namespace) + + +def update_dynamic_action(identifier, values, namespace=''): + return IMPL.update_dynamic_action(identifier, values, namespace) + + +def get_dynamic_action(identifier, namespace='', fields=()): + return IMPL.get_dynamic_action(identifier, fields, namespace) + + +def get_dynamic_actions(limit=None, marker=None, sort_keys=None, + sort_dirs=None, fields=None, **kwargs): + return IMPL.get_dynamic_actions( + limit=limit, + marker=marker, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + fields=fields, + **kwargs + ) + + +def update_code_source(identifier, values, namespace=''): + return IMPL.update_code_source(identifier, values, namespace=namespace) + + +def get_code_sources(limit=None, marker=None, sort_keys=None, + sort_dirs=None, fields=None, **kwargs): + return IMPL.get_code_sources( + limit=limit, + marker=marker, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + fields=fields, + **kwargs + ) + + +def delete_code_source(name, namespace=''): + return IMPL.delete_code_source(name, namespace=namespace) + + +def get_code_source(identifier, namespace='', fields=()): + return IMPL.get_code_source(identifier, fields, namespace=namespace) + + +def create_code_source(values): + return IMPL.create_code_source(values) + def get_action_definition_by_id(id, fields=()): return IMPL.get_action_definition_by_id(id, fields=fields) diff --git a/mistral/db/v2/sqlalchemy/api.py b/mistral/db/v2/sqlalchemy/api.py index 33c5b6d68..f964f0d2f 100644 --- a/mistral/db/v2/sqlalchemy/api.py +++ b/mistral/db/v2/sqlalchemy/api.py @@ -285,6 +285,13 @@ def _get_collection(model, insecure=False, limit=None, marker=None, return query.all() +def get_db_objects(model, insecure=False, **filters): + query = b.model_query(model) if insecure else _secure_query(model) + query = db_filters.apply_filters(query, model, **filters) + + return query + + def _get_count(model, insecure=False, **filters): query = b.model_query(model) if insecure else _secure_query(model) @@ -635,6 +642,133 @@ def delete_workflow_definitions(session=None, **kwargs): # Action definitions. +@b.session_aware() +def create_code_source(values, session=None): + code_src = models.CodeSource() + code_src.update(values.copy()) + + try: + code_src.save(session=session) + except db_exc.DBDuplicateEntry: + raise exc.DBDuplicateEntryError( + "Duplicate entry for CodeSource ['name', 'namespace'," + " 'project_id']: {}, {}, {}".format(code_src.name, + code_src.namespace, + code_src.project_id) + ) + + return code_src + + +@b.session_aware() +def get_code_sources(fields=None, session=None, **kwargs): + return _get_collection( + model=models.CodeSource, + fields=fields, + **kwargs + ) + + +@b.session_aware() +def get_code_source(identifier, fields=(), session=None, namespace=''): + code_src = _get_db_object_by_name_and_namespace_or_id( + models.CodeSource, + identifier, + namespace=namespace, + columns=fields + ) + + if not code_src: + raise exc.DBEntityNotFoundError( + "Code Source not found [name=%s,namespace=%s]" + % (identifier, namespace) + ) + + return code_src + + +@b.session_aware() +def update_code_source(identifier, values, namespace='', session=None): + code_src = get_code_source(identifier, namespace=namespace) + + values['version'] = code_src.version + 1 + + code_src.update(values.copy()) + + return code_src + + +@b.session_aware() +def delete_code_source(identifier, namespace='', session=None): + code_src = get_code_source( + identifier, + namespace=namespace, + session=session + ) + + session.delete(code_src) + + +@b.session_aware() +def create_dynamic_action(values, session=None): + action_def = models.DynamicAction() + + action_def.update(values.copy()) + + try: + action_def.save(session=session) + except db_exc.DBDuplicateEntry: + raise exc.DBDuplicateEntryError( + "Duplicate entry for Action[name=%s, namespace=%s, project_id=%s]" + % (action_def.name, action_def.namespace, action_def.project_id) + ) + + return action_def + + +@b.session_aware() +def update_dynamic_action(identifier, values, namespace='', session=None): + action_def = get_dynamic_action(identifier, namespace=namespace) + + action_def.update(values.copy()) + + return action_def + + +@b.session_aware() +def get_dynamic_action(identifier, fields=(), namespace='', session=None): + action = _get_db_object_by_name_and_namespace_or_id( + models.DynamicAction, + identifier, + namespace=namespace, + columns=fields + ) + + if not action: + raise exc.DBEntityNotFoundError( + "Dynamic Action not found [name=%s,namespace=%s]" + % (identifier, namespace) + ) + + return action + + +@b.session_aware() +def get_dynamic_actions(fields=None, session=None, **kwargs): + return _get_collection( + model=models.DynamicAction, + fields=fields, + **kwargs + ) + + +@b.session_aware() +def delete_dynamic_action(identifier, namespace='', session=None): + action_def = get_dynamic_action(identifier, namespace) + print(action_def) + session.delete(action_def) + + @b.session_aware() def get_action_definition_by_id(id, fields=(), session=None): action_def = _get_db_object_by_id( @@ -671,7 +805,7 @@ def get_action_definition(identifier, fields=(), session=None, namespace=''): if not a_def: raise exc.DBEntityNotFoundError( - "Action definition not found [action_name=%s,namespace=%s]" + "Action definition not found [action_name=%s, namespace=%s]" % (identifier, namespace) ) diff --git a/mistral/db/v2/sqlalchemy/models.py b/mistral/db/v2/sqlalchemy/models.py index 9f234689d..fa31d83c6 100644 --- a/mistral/db/v2/sqlalchemy/models.py +++ b/mistral/db/v2/sqlalchemy/models.py @@ -196,8 +196,58 @@ class ActionDefinition(Definition): attributes = sa.Column(st.JsonDictType()) +class CodeSource(mb.MistralSecureModelBase): + """Contains info about registered CodeSources.""" + + __tablename__ = 'code_sources' + + id = mb.id_column() + name = sa.Column(sa.String(255)) + src = sa.Column(sa.Text()) + version = sa.Column(sa.Integer()) + namespace = sa.Column(sa.String(255), nullable=True) + tags = sa.Column(st.JsonListType()) + + __table_args__ = ( + sa.UniqueConstraint( + 'name', + 'namespace', + 'project_id'), + + sa.Index('%s_project_id' % __tablename__, 'project_id'), + sa.Index('%s_scope' % __tablename__, 'scope'), + ) + + +class DynamicAction(mb.MistralSecureModelBase): + """Contains info about registered Dynamic Actions.""" + + __tablename__ = 'dynamic_actions' + + # Main properties. + id = mb.id_column() + name = sa.Column(sa.String(255)) + namespace = sa.Column(sa.String(255), nullable=True) + class_name = sa.Column(sa.String(255)) + __table_args__ = ( + sa.UniqueConstraint( + 'name', + 'namespace', + 'project_id'), + sa.Index('%s_project_id' % __tablename__, 'project_id'), + sa.Index('%s_scope' % __tablename__, 'scope'), + ) + + +DynamicAction.code_source_id = sa.Column( + sa.String(36), + sa.ForeignKey(CodeSource.id, ondelete='CASCADE'), + nullable=False +) + # Execution objects. + class Execution(mb.MistralSecureModelBase): __abstract__ = True diff --git a/mistral/policies/__init__.py b/mistral/policies/__init__.py index a0e8aa9f7..e708da6be 100644 --- a/mistral/policies/__init__.py +++ b/mistral/policies/__init__.py @@ -17,7 +17,9 @@ import itertools from mistral.policies import action from mistral.policies import action_executions from mistral.policies import base +from mistral.policies import code_sources from mistral.policies import cron_trigger +from mistral.policies import dynamic_actions from mistral.policies import environment from mistral.policies import event_trigger from mistral.policies import execution @@ -33,6 +35,8 @@ def list_rules(): action.list_rules(), action_executions.list_rules(), base.list_rules(), + code_sources.list_rules(), + dynamic_actions.list_rules(), cron_trigger.list_rules(), environment.list_rules(), event_trigger.list_rules(), diff --git a/mistral/policies/code_sources.py b/mistral/policies/code_sources.py new file mode 100644 index 000000000..39328988e --- /dev/null +++ b/mistral/policies/code_sources.py @@ -0,0 +1,83 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_policy import policy + +from mistral.policies import base + +CODE_SOURCES = 'code_sources:%s' +BASE_PATH = '/v2/code_sources' + +rules = [ + policy.DocumentedRuleDefault( + name=CODE_SOURCES % 'create', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Create a new code source.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'POST' + } + ] + ), + policy.DocumentedRuleDefault( + name=CODE_SOURCES % 'delete', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Delete the named code source.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=CODE_SOURCES % 'get', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Return the named code source.', + operations=[ + { + 'path': BASE_PATH + '/{action_id}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=CODE_SOURCES % 'list', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Return all code sources.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=CODE_SOURCES % 'update', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Update one or more code source.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'PUT' + } + ] + ) +] + + +def list_rules(): + return rules diff --git a/mistral/policies/dynamic_actions.py b/mistral/policies/dynamic_actions.py new file mode 100644 index 000000000..e976672e0 --- /dev/null +++ b/mistral/policies/dynamic_actions.py @@ -0,0 +1,83 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_policy import policy + +from mistral.policies import base + +ACTIONS = 'dynamic_actions:%s' +BASE_PATH = '/v2/dynamic_actions' + +rules = [ + policy.DocumentedRuleDefault( + name=ACTIONS % 'create', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Create a new dynamic action.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'POST' + } + ] + ), + policy.DocumentedRuleDefault( + name=ACTIONS % 'delete', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Delete the named dynamic action.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=ACTIONS % 'get', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Return the named dynamic action.', + operations=[ + { + 'path': BASE_PATH + '/{action_id}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=ACTIONS % 'list', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Return all dynamic actions.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=ACTIONS % 'update', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Update one or more dynamic actions.', + operations=[ + { + 'path': BASE_PATH, + 'method': 'PUT' + } + ] + ) +] + + +def list_rules(): + return rules diff --git a/mistral/services/actions.py b/mistral/services/actions.py index 637b1f020..dc9da6abe 100644 --- a/mistral/services/actions.py +++ b/mistral/services/actions.py @@ -25,7 +25,6 @@ from mistral_lib import actions as ml_actions from mistral.actions import test - LOG = logging.getLogger(__name__) _SYSTEM_PROVIDER = None diff --git a/mistral/services/code_sources.py b/mistral/services/code_sources.py new file mode 100644 index 000000000..347f16727 --- /dev/null +++ b/mistral/services/code_sources.py @@ -0,0 +1,97 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from oslo_log import log as logging + +from mistral.db.v2 import api as db_api + +LOG = logging.getLogger(__name__) + +_SYSTEM_PROVIDER = None +_TEST_PROVIDER = None + + +def create_code_source(name, src_code, namespace='', version=1): + with db_api.transaction(): + return db_api.create_code_source({ + 'name': name, + 'namespace': namespace, + 'version': version, + 'src': src_code, + }) + + +def create_code_sources(namespace='', **files): + return _update_or_create_code_sources( + create_code_source, + namespace, + **files + ) + + +def _update_or_create_code_sources(operation, namespace='', **files): + code_sources = [] + + for file in files: + filename = files[file].name + file_content = files[file].file.read().decode() + + code_sources.append( + operation( + filename, + file_content, + namespace + ) + ) + + return code_sources + + +def update_code_sources(namespace='', **files): + return _update_or_create_code_sources( + update_code_source, + namespace, + **files + ) + + +def update_code_source(identifier, src_code, namespace=''): + with db_api.transaction(): + return db_api.update_code_source( + identifier=identifier, + namespace=namespace, + values={ + 'src': src_code, + } + ) + + +def delete_code_source(identifier, namespace=''): + with db_api.transaction(): + db_api.delete_code_source(identifier, namespace=namespace) + + +def delete_code_sources(code_sources, namespace=''): + with db_api.transaction(): + for code_source in code_sources: + db_api.delete_code_source(code_source, namespace=namespace) + + +def get_code_source(identifier, namespace='', fields=()): + return db_api.get_code_source( + identifier, + namespace=namespace, + fields=fields + ) diff --git a/mistral/services/dynamic_actions.py b/mistral/services/dynamic_actions.py new file mode 100644 index 000000000..234a17c52 --- /dev/null +++ b/mistral/services/dynamic_actions.py @@ -0,0 +1,82 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from oslo_log import log as logging + +from mistral.db.v2 import api as db_api + + +LOG = logging.getLogger(__name__) + + +def create_dynamic_actions(action_list, namespace=''): + created_actions = [] + with db_api.transaction(): + for action in action_list: + created_actions.append( + db_api.create_dynamic_action({ + 'name': action['name'], + 'class_name': action['class_name'], + 'namespace': namespace, + 'code_source_id': action['code_source_id'] + }) + ) + + return created_actions + + +def delete_dynamic_action(identifier, namespace=''): + with db_api.transaction(): + return db_api.delete_dynamic_action( + identifier, + namespace + ) + + +def get_dynamic_actions(limit=None, marker=None, sort_keys=None, + sort_dirs=None, fields=None, **kwargs): + with db_api.transaction(): + return db_api.get_dynamic_actions( + limit=limit, + marker=marker, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + fields=fields, + **kwargs + ) + + +def get_dynamic_action(identifier, namespace=''): + with db_api.transaction(): + return db_api.get_dynamic_action( + identifier, + namespace=namespace + ) + + +def update_dynamic_action(identifier, values, namespace=''): + with db_api.transaction(): + return db_api.update_dynamic_action( + identifier, + values, + namespace + ) + + +def update_dynamic_actions(actions, namespace=''): + return [ + update_dynamic_action(name, values, namespace) + for name, values in actions.items() + ] diff --git a/mistral/tests/unit/actions/test_dynamic_action_provider.py b/mistral/tests/unit/actions/test_dynamic_action_provider.py new file mode 100644 index 000000000..1484e6c26 --- /dev/null +++ b/mistral/tests/unit/actions/test_dynamic_action_provider.py @@ -0,0 +1,126 @@ +# Copyright 2020 Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.actions import dynamic_action +from mistral.services import code_sources as code_sources_service +from mistral.services import dynamic_actions as dynamic_actions_service +from mistral.tests.unit import base + +DUMMY_CODE_SOURCE = """from mistral_lib import actions + +class DummyAction(actions.Action): + def run(self, context): + return None + def test(self, context): + return None +class DummyAction2(actions.Action): + def run(self, context): + return None + def test(self, context): + return None +""" + +ACTIONS = [] +NAMESPACE = "ns" + + +class DynamicActionProviderTest(base.DbTestCase): + + def _create_code_source(self, namespace=''): + return code_sources_service.create_code_source( + name='code_source', + src_code=DUMMY_CODE_SOURCE, + namespace=namespace + ) + + def _delete_code_source(self): + return code_sources_service.delete_code_source( + identifier='code_source', + ) + + def _create_dynamic_actions(self, code_source_id, namespace=''): + actions = [ + { + "name": "dummy_action", + "class_name": "DummyAction", + "code_source_id": code_source_id + }, + { + "name": "dummy_action2", + "class_name": "DummyAction2", + "code_source_id": code_source_id + }] + + dynamic_actions_service.create_dynamic_actions( + actions, + namespace=namespace + ) + + def test_Dynamic_actions(self): + provider = dynamic_action.DynamicActionProvider() + + action_descs = provider.find_all() + + self.assertEqual(0, len(action_descs)) + + code_source = self._create_code_source() + self._create_dynamic_actions(code_source_id=code_source['id']) + + action_descs = provider.find_all() + + self.assertEqual(2, len(action_descs)) + + self._delete_code_source() + + def test_loaded_actions_deleted_from_db(self): + provider = dynamic_action.DynamicActionProvider() + + action_descs = provider.find_all() + + self.assertEqual(0, len(action_descs)) + + code_source = self._create_code_source() + self._create_dynamic_actions(code_source_id=code_source['id']) + + action_descs = provider.find_all() + + self.assertEqual(2, len(action_descs)) + + self._delete_code_source() + + action_descs = provider.find_all() + + self.assertEqual(0, len(action_descs)) + + def test_Dynamic_actions_with_namespace(self): + provider = dynamic_action.DynamicActionProvider() + + action_descs = provider.find_all() + + self.assertEqual(0, len(action_descs)) + + code_source = self._create_code_source() + self._create_dynamic_actions( + code_source_id=code_source['id'], + namespace=NAMESPACE + ) + action_descs = provider.find_all(namespace=NAMESPACE) + + self.assertEqual(2, len(action_descs)) + + action_descs = provider.find_all(namespace='') + + self.assertEqual(0, len(action_descs)) + + self._delete_code_source() diff --git a/mistral/tests/unit/api/v2/test_code_sources.py b/mistral/tests/unit/api/v2/test_code_sources.py new file mode 100644 index 000000000..44ddb876d --- /dev/null +++ b/mistral/tests/unit/api/v2/test_code_sources.py @@ -0,0 +1,164 @@ +# Copyright 2020 - Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from mistral.tests.unit.api import base + +FILE_CONTENT = """test file""" + +UPDATED_FILE_CONTENT = """updated content""" + +MODULE_NAME = 'modulename%s' +NAMESPACE = "NS" + + +class TestCodeSourcesController(base.APITest): + + def _create_code_source(self, module_name, file_content, + namespace=NAMESPACE, expect_errors=False): + return self.app.post( + '/v2/code_sources', + params={'namespace': namespace}, + upload_files=[ + (module_name, 'filename', file_content.encode()) + ], + expect_errors=expect_errors + ) + + def _delete_code_source(self, id, namespace=NAMESPACE): + return self.app.delete( + '/v2/code_sources/%s?namespace=%s' % (id, namespace) + ) + + def test_create_code_source(self): + mod_name = MODULE_NAME % 'create' + + resp = self._create_code_source( + mod_name, + FILE_CONTENT) + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + code_sources = resp_json.get('code_sources') + + self.assertEqual(1, len(code_sources)) + + code_source = code_sources[0] + + self.assertEqual(mod_name, code_source.get('name')) + self.assertEqual(FILE_CONTENT, code_source.get('src')) + self.assertEqual(1, code_source.get('version')) + self.assertEqual(NAMESPACE, code_source.get('namespace')) + + self._delete_code_source(mod_name) + + def test_update_code_source(self): + mod_name = MODULE_NAME % 'update' + + self._create_code_source(mod_name, FILE_CONTENT) + resp = self.app.put( + '/v2/code_sources/', + params='namespace=%s' % NAMESPACE, + upload_files=[ + (mod_name, 'filename', UPDATED_FILE_CONTENT.encode()) + ], + ) + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + code_sources = resp_json.get('code_sources') + + self.assertEqual(1, len(code_sources)) + + code_source = code_sources[0] + + self.assertEqual(200, resp.status_int) + + self.assertEqual(mod_name, code_source.get('name')) + self.assertEqual(UPDATED_FILE_CONTENT, code_source.get('src')) + self.assertEqual(2, code_source.get('version')) + self.assertEqual(NAMESPACE, code_source.get('namespace')) + + self._delete_code_source(mod_name) + + def test_delete_code_source(self): + mod_name = MODULE_NAME % 'delete' + resp = self._create_code_source(mod_name, FILE_CONTENT) + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + code_sources = resp_json.get('code_sources') + + self.assertEqual(1, len(code_sources)) + + self._delete_code_source(mod_name) + + def test_create_duplicate_code_source(self): + mod_name = MODULE_NAME % 'duplicate' + self._create_code_source(mod_name, FILE_CONTENT) + resp = self._create_code_source( + mod_name, + FILE_CONTENT, expect_errors=True + ) + + self.assertEqual(409, resp.status_int) + self.assertIn('Duplicate entry for CodeSource', resp) + self._delete_code_source(mod_name) + + def test_get_code_source(self): + mod_name = MODULE_NAME % 'get' + self._create_code_source(mod_name, FILE_CONTENT) + + resp = self.app.get( + '/v2/code_sources/%s' % mod_name, + params='namespace=%s' % NAMESPACE + ) + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + self.assertEqual(mod_name, resp_json.get('name')) + self.assertEqual(FILE_CONTENT, resp_json.get('src')) + self.assertEqual(1, resp_json.get('version')) + self.assertEqual(NAMESPACE, resp_json.get('namespace')) + + self._delete_code_source(mod_name) + + def test_get_all_code_source(self): + mod_name = MODULE_NAME % 'getall' + mod2_name = MODULE_NAME % '2getall' + + self._create_code_source(mod_name, FILE_CONTENT) + self._create_code_source(mod2_name, FILE_CONTENT) + + resp = self.app.get( + '/v2/code_sources', + params='namespace=%s' % NAMESPACE + ) + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + code_sources = resp_json.get('code_sources') + + self.assertEqual(2, len(code_sources)) + + self._delete_code_source(mod_name) + self._delete_code_source(mod2_name) diff --git a/mistral/tests/unit/api/v2/test_dynamic_actions.py b/mistral/tests/unit/api/v2/test_dynamic_actions.py new file mode 100644 index 000000000..9d0625259 --- /dev/null +++ b/mistral/tests/unit/api/v2/test_dynamic_actions.py @@ -0,0 +1,174 @@ +# Copyright 2020 - Nokia Software. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from mistral.tests.unit.api import base + +FILE_CONTENT = """from mistral_lib import actions + +class DummyAction(actions.Action): + def run(self, context): + return None + + def test(self, context): + return None + +class DummyAction2(actions.Action): + def run(self, context): + return None + + def test(self, context): + return None""" + +CREATE_REQUEST = """ +- + name: dummy_action + class_name: DummyAction + code_source_id: {} +""" + +UPDATE_REQUEST = """ +dummy_action: + class_name: NewDummyAction + code_source_id: {} +""" + + +class TestDynamicActionsController(base.APITest): + + def setUp(self): + super(TestDynamicActionsController, self).setUp() + resp = self._create_code_source().json + + self.code_source_id = resp.get('code_sources')[0].get('id') + self.addCleanup(self._delete_code_source) + + def _create_code_source(self): + return self.app.post( + '/v2/code_sources', + upload_files=[ + ('modulename', 'filename', FILE_CONTENT.encode()) + ], + ) + + def _create_dynamic_action(self, body): + return self.app.post( + '/v2/dynamic_actions', + body, + content_type="text/plain" + ) + + def _delete_code_source(self): + return self.app.delete( + '/v2/code_sources/modulename', + ) + + def test_create_dynamic_action(self): + resp = self._create_dynamic_action( + CREATE_REQUEST.format(self.code_source_id) + ) + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + dynamic_actions = resp_json.get('dynamic_actions') + + self.assertEqual(1, len(dynamic_actions)) + + dynamic_action = dynamic_actions[0] + + self.assertEqual('dummy_action', dynamic_action.get('name')) + self.assertEqual('DummyAction', dynamic_action.get('class_name')) + self.assertEqual( + self.code_source_id, + dynamic_action.get('code_source_id') + ) + self.app.delete('/v2/dynamic_actions/dummy_action') + + def test_update_dynamic_action(self): + self._create_dynamic_action( + CREATE_REQUEST.format(self.code_source_id) + ) + + resp = self.app.put( + '/v2/dynamic_actions', + UPDATE_REQUEST.format(self.code_source_id), + content_type="text/plain" + ) + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + dynamic_actions = resp_json.get('dynamic_actions') + + self.assertEqual(1, len(dynamic_actions)) + + dynamic_action = dynamic_actions[0] + + self.assertEqual('dummy_action', dynamic_action.get('name')) + self.assertEqual('NewDummyAction', dynamic_action.get('class_name')) + self.assertEqual( + self.code_source_id, + dynamic_action.get('code_source_id') + ) + + self.app.delete('/v2/dynamic_actions/dummy_action') + + def test_get_dynamic_action(self): + resp = self._create_dynamic_action( + CREATE_REQUEST.format(self.code_source_id) + ) + + self.assertEqual(200, resp.status_int) + + self.app.delete('/v2/dynamic_actions/dummy_action') + + def test_get_all_dynamic_actions(self): + self._create_dynamic_action( + CREATE_REQUEST.format(self.code_source_id) + ) + + resp = self.app.get('/v2/dynamic_actions') + + resp_json = resp.json + + self.assertEqual(200, resp.status_int) + + dynamic_actions = resp_json.get('dynamic_actions') + + self.assertEqual(1, len(dynamic_actions)) + + self.app.delete('/v2/dynamic_actions/dummy_action') + + def test_delete_dynamic_action(self): + resp = self._create_dynamic_action( + CREATE_REQUEST.format(self.code_source_id) + ) + + self.assertEqual(200, resp.status_int) + + resp = self.app.get('/v2/dynamic_actions/dummy_action') + + self.assertEqual(200, resp.status_int) + + self.app.delete('/v2/dynamic_actions/dummy_action') + + resp = self.app.get( + '/v2/dynamic_actions/dummy_action', + expect_errors=True + ) + + self.assertEqual(404, resp.status_int) diff --git a/releasenotes/notes/add_dynamic_actions.yaml b/releasenotes/notes/add_dynamic_actions.yaml new file mode 100644 index 000000000..2574a16ac --- /dev/null +++ b/releasenotes/notes/add_dynamic_actions.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Now users can upload Python code through the API (code_sources API) + and create actions from it dynamically (using dynamic_actions API). + If needed, actions can be also modified and deleted. + Note that this all doesn't require a Mistral restart. + - | + Added a new endpoint "/v2/code_sources/", this is used to create, + update, delete and get code sources from mistral. + - | + Added a new endpoint "/v2/dynamic_actions/", this is used to create, + update, delete and get dynamic actions from mistral runtime. diff --git a/setup.cfg b/setup.cfg index 543b5cad8..31a0dc7fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ oslo.policy.enforcer = mistral.action.providers = legacy = mistral.actions.legacy:LegacyActionProvider adhoc = mistral.actions.adhoc:AdHocActionProvider + dynamic = mistral.actions.dynamic_action:DynamicActionProvider mistral.actions = std.async_noop = mistral.actions.std_actions:AsyncNoOpAction