Decoupling of Mistral tempest test from Mistral code base

The Mistral Tempest tests have a hard-coded dependency on Mistral being present
when Tempest tests are executed.
When trying to sparse-checkout the mistral_tempest_tests folder to run
the Mistral tests as a Tempest plugin; it fails due to Mistral not
being installed as some utilities and resources which are written in
the Mistral Tempest tests are being hard referenced from Mistral
being installed in the same environment.

This patch decouples the Mistral Tempest Tests so that they can be executed in
a stand-alone mode along with the necessary resources that are required to
execute the Tempest tests.

Change-Id: Ifd6a3a65a14c4ad4736dccc3e72cd564b6f53a0a
Closes-Bug: #1714732

mistral_tempest_tests/services/base.py

@@ -30,7 +30,9 @@ def get_resource(path):
     main_package = 'mistral_tempest_tests'
     dir_path = __file__[0:__file__.find(main_package)]
 
-    return open(dir_path + 'mistral/tests/resources/' + path).read()
+    return open(dir_path +
+                'mistral_tempest_tests/tests/resources/' +
+                path).read()
 
 
 def find_items(items, **props):

mistral_tempest_tests/tests/api/v2/test_actions.py

@@ -16,8 +16,8 @@ import datetime
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 
 class ActionTestsV2(base.TestCase):

mistral_tempest_tests/tests/api/v2/test_executions.py

@@ -16,8 +16,8 @@ from oslo_concurrency.fixture import lockutils
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 import json
 

mistral_tempest_tests/tests/api/v2/test_workflows.py

@@ -17,8 +17,8 @@ from oslo_concurrency.fixture import lockutils
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 
 class WorkflowTestsV2(base.TestCase):

mistral_tempest_tests/tests/resources/action_v2.yaml

@@ -0,0 +1,21 @@
+---
+version: "2.0"
+
+greeting:
+  description: "This action says 'Hello'"
+  tags: [hello]
+  base: std.echo
+  base-input:
+    output: 'Hello,  <% $.name %>'
+  input:
+    - name
+  output:
+    string: <% $ %>
+
+farewell:
+  base: std.echo
+  base-input:
+    output: 'Bye!'
+  output:
+    info: <% $ %>
+

mistral_tempest_tests/tests/resources/for_wf_namespace/lowest_level_wf.yaml

@@ -0,0 +1,6 @@
+---
+version: '2.0'
+lowest_level_wf:
+  tasks:
+    noop_task:
+      action: std.noop

mistral_tempest_tests/tests/resources/for_wf_namespace/middle_wf.yaml

@@ -0,0 +1,6 @@
+---
+version: '2.0'
+middle_wf:
+  tasks:
+    run_workflow_with_name_lowest_level_wf:
+      workflow: lowest_level_wf

mistral_tempest_tests/tests/resources/for_wf_namespace/top_level_wf.yaml

@@ -0,0 +1,6 @@
+---
+version: '2.0'
+top_level_wf:
+  tasks:
+    run_workflow_with_name_middle_wf:
+      workflow: middle_wf

mistral_tempest_tests/tests/resources/openstack/action_collection_wb.yaml

@@ -0,0 +1,53 @@
+---
+version: '2.0'
+name: action_collection
+
+workflows:
+  keystone:
+    type: direct
+    tasks:
+      projects_list:
+        action: keystone.projects_list
+        publish:
+           result: <% task().result %>
+
+  nova:
+    type: direct
+    tasks:
+      flavors_list:
+        action: nova.flavors_list
+        publish:
+           result: <% task().result %>
+
+  glance:
+    type: direct
+    tasks:
+      images_list:
+        action: glance.images_list
+        publish:
+           result: <% task().result %>
+
+  heat:
+    type: direct
+    tasks:
+      stacks_list:
+        action: heat.stacks_list
+        publish:
+           result: <% task().result %>
+
+  neutron:
+    type: direct
+    tasks:
+      list_subnets:
+        action: neutron.list_subnets
+        publish:
+           result: <% task().result %>
+
+  cinder:
+    type: direct
+    tasks:
+      volumes_list:
+        action: cinder.volumes_list
+        publish:
+           result: <% task().result %>
+

mistral_tempest_tests/tests/resources/single_wf.yaml

@@ -0,0 +1,11 @@
+---
+version: '2.0'
+
+single_wf:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Hello"
+      publish:
+        result: <% task(hello).result %>

mistral_tempest_tests/tests/resources/wb_v1.yaml

@@ -0,0 +1,12 @@
+Namespaces:
+  Greetings:
+    actions:
+      hello:
+        class: std.echo
+        base-parameters:
+          output: Hello!
+
+Workflow:
+  tasks:
+    hello:
+      action: Greetings.hello

mistral_tempest_tests/tests/resources/wb_v2.yaml

@@ -0,0 +1,13 @@
+---
+version: '2.0'
+name: test
+
+workflows:
+  test:
+    type: direct
+
+    tasks:
+      hello:
+        action: std.echo output="Hello"
+        publish:
+           result: <% task(hello).result %>

mistral_tempest_tests/tests/resources/wb_with_nested_wf.yaml

@@ -0,0 +1,18 @@
+---
+version: "2.0"
+
+name: wb_with_nested_wf
+
+workflows:
+
+  wrapping_wf:
+    type: direct
+    tasks:
+      call_inner_wf:
+        workflow: inner_wf
+
+  inner_wf:
+    type: direct
+    tasks:
+      hello:
+        action: std.echo output="Hello from inner workflow"

mistral_tempest_tests/tests/resources/wf_action_ex_concurrency.yaml

@@ -0,0 +1,8 @@
+---
+version: '2.0'
+
+test_action_ex_concurrency:
+  tasks:
+    test_with_items:
+      with-items: index in <% range(2) %>
+      action: std.echo output='<% $.index %>'

mistral_tempest_tests/tests/resources/wf_task_ex_concurrency.yaml

@@ -0,0 +1,11 @@
+---
+version: '2.0'
+
+test_task_ex_concurrency:
+  tasks:
+    task1:
+      action: std.async_noop
+      timeout: 2
+    task2:
+      action: std.async_noop
+      timeout: 2

mistral_tempest_tests/tests/resources/wf_v2.yaml

@@ -0,0 +1,34 @@
+---
+version: '2.0'
+
+wf:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Hello"
+      wait-before: 1
+      publish:
+        result: <% task(hello).result %>
+
+wf1:
+  type: reverse
+  input:
+    - farewell
+
+  tasks:
+    addressee:
+      action: std.echo output="John"
+      publish:
+        name: <% task(addressee).result %>
+
+    goodbye:
+      action: std.echo output="<% $.farewell %>, <% $.name %>"
+      requires: [addressee]
+
+wf2:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Doe"

mistral_tempest_tests/tests/scenario/engine/actions/v2/test_ssh_actions.py

@@ -23,9 +23,9 @@ from tempest import config
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
-from mistral.utils import ssh_utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import ssh_utils
+from mistral_tempest_tests.tests import utils
 
 
 LOG = logging.getLogger(__name__)

mistral_tempest_tests/tests/ssh_utils.py

@@ -0,0 +1,103 @@
+# Copyright 2014 - Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from os import path
+from oslo_log import log as logging
+import paramiko
+import six
+
+KEY_PATH = path.expanduser("~/.ssh/")
+LOG = logging.getLogger(__name__)
+
+
+def _read_paramimko_stream(recv_func):
+    result = ''
+    buf = recv_func(1024)
+    while buf != '':
+        result += buf
+        buf = recv_func(1024)
+
+    return result
+
+
+def _to_paramiko_private_key(private_key_filename, password=None):
+    if '../' in private_key_filename or '..\\' in private_key_filename:
+        raise OSError(
+            "Private key filename must not contain '..'. "
+            "Actual: %s" % private_key_filename
+        )
+
+    private_key_path = KEY_PATH + private_key_filename
+
+    return paramiko.RSAKey(
+        filename=private_key_path,
+        password=password
+    )
+
+
+def _connect(host, username, password=None, pkey=None, proxy=None):
+    if isinstance(pkey, six.string_types):
+        pkey = _to_paramiko_private_key(pkey, password)
+
+    LOG.debug('Creating SSH connection to %s', host)
+
+    ssh_client = paramiko.SSHClient()
+    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh_client.connect(
+        host,
+        username=username,
+        password=password,
+        pkey=pkey,
+        sock=proxy
+    )
+
+    return ssh_client
+
+
+def _cleanup(ssh_client):
+    ssh_client.close()
+
+
+def _execute_command(ssh_client, cmd, get_stderr=False,
+                     raise_when_error=True):
+    try:
+        chan = ssh_client.get_transport().open_session()
+        chan.exec_command(cmd)
+
+        # TODO(nmakhotkin): that could hang if stderr buffer overflows
+        stdout = _read_paramimko_stream(chan.recv)
+        stderr = _read_paramimko_stream(chan.recv_stderr)
+
+        ret_code = chan.recv_exit_status()
+
+        if ret_code and raise_when_error:
+            raise RuntimeError("Cmd: %s\nReturn code: %s\nstdout: %s"
+                               % (cmd, ret_code, stdout))
+        if get_stderr:
+            return ret_code, stdout, stderr
+        else:
+            return ret_code, stdout
+    finally:
+        _cleanup(ssh_client)
+
+
+def execute_command(cmd, host, username, password=None,
+                    private_key_filename=None, get_stderr=False,
+                    raise_when_error=True):
+    ssh_client = _connect(host, username, password, private_key_filename)
+
+    LOG.debug("Executing command %s", cmd)
+
+    return _execute_command(ssh_client, cmd, get_stderr, raise_when_error)

mistral_tempest_tests/tests/utils.py

@@ -0,0 +1,152 @@
+# Copyright 2013 - Mirantis, Inc.
+# Copyright 2015 - Huawei Technologies Co. Ltd
+# Copyright 2016 - Brocade Communications Systems, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import contextlib
+import json
+import os
+import shutil
+import tempfile
+
+from oslo_concurrency import processutils
+
+
+class NotDefined(object):
+    """Marker of an empty value.
+
+    In a number of cases None can't be used to express the semantics of
+    a not defined value because None is just a normal value rather than
+    a value set to denote that it's not defined. This class can be used
+    in such cases instead of None.
+    """
+
+    pass
+
+
+def get_dict_from_string(string, delimiter=','):
+    if not string:
+        return {}
+
+    kv_dicts = []
+
+    for kv_pair_str in string.split(delimiter):
+        kv_str = kv_pair_str.strip()
+        kv_list = kv_str.split('=')
+
+        if len(kv_list) > 1:
+            try:
+                value = json.loads(kv_list[1])
+            except ValueError:
+                value = kv_list[1]
+
+            kv_dicts += [{kv_list[0]: value}]
+        else:
+            kv_dicts += [kv_list[0]]
+
+    return get_dict_from_entries(kv_dicts)
+
+
+def get_dict_from_entries(entries):
+    """Transforms a list of entries into dictionary.
+
+    :param entries: A list of entries.
+        If an entry is a dictionary the method simply updates the result
+        dictionary with its content.
+        If an entry is not a dict adds {entry, NotDefined} into the result.
+    """
+
+    result = {}
+
+    for e in entries:
+        if isinstance(e, dict):
+            result.update(e)
+        else:
+            # NOTE(kong): we put NotDefined here as the value of
+            # param without value specified, to distinguish from
+            # the valid values such as None, ''(empty string), etc.
+            result[e] = NotDefined
+
+    return result
+
+
+@contextlib.contextmanager
+def tempdir(**kwargs):
+    argdict = kwargs.copy()
+
+    if 'dir' not in argdict:
+        argdict['dir'] = '/tmp/'
+
+    tmpdir = tempfile.mkdtemp(**argdict)
+
+    try:
+        yield tmpdir
+    finally:
+        try:
+            shutil.rmtree(tmpdir)
+        except OSError as e:
+            raise OSError(
+                "Failed to delete temp dir %(dir)s (reason: %(reason)s)" %
+                {'dir': tmpdir, 'reason': e}
+            )
+
+
+def save_text_to(text, file_path, overwrite=False):
+    if os.path.exists(file_path) and not overwrite:
+        raise OSError(
+            "Cannot save data to file. File %s already exists."
+        )
+
+    with open(file_path, 'w') as f:
+        f.write(text)
+
+
+def generate_key_pair(key_length=2048):
+    """Create RSA key pair with specified number of bits in key.
+
+    Returns tuple of private and public keys.
+    """
+    with tempdir() as tmpdir:
+        keyfile = os.path.join(tmpdir, 'tempkey')
+        args = [
+            'ssh-keygen',
+            '-q',  # quiet
+            '-N', '',  # w/o passphrase
+            '-t', 'rsa',  # create key of rsa type
+            '-f', keyfile,  # filename of the key file
+            '-C', 'Generated-by-Mistral'  # key comment
+        ]
+
+        if key_length is not None:
+            args.extend(['-b', key_length])
+
+        processutils.execute(*args)
+
+        if not os.path.exists(keyfile):
+            # raise exc.DataAccessException(
+            #     "Private key file hasn't been created"
+            # )
+            raise OSError("Private key file hasn't been created")
+
+        private_key = open(keyfile).read()
+        public_key_path = keyfile + '.pub'
+
+        if not os.path.exists(public_key_path):
+            # raise exc.DataAccessException(
+            #     "Public key file hasn't been created"
+            # )
+            raise OSError("Private key file hasn't been created")
+        public_key = open(public_key_path).read()
+
+        return private_key, public_key