Support classes for heat integration tests

These support classes started as a forklift of the classes needed
to run tempest scenario orchestration tests.

The original tempest code has been pared back to provide the
small subset required by heat integration tests. From this point on
these support classes can evolve to the specific needs of the
integration tests.

There is some unused code (especially in remote_client) which has
been left in as it may become useful in the future, and is already
extremely well reviewed and tested from being developed for tempest.

The script heat_integrationtests/generate_sample.sh will generate
an up-to-date heat_integrationtests/heat_integrationtests.conf.sample
file which can be copied to
heat_integrationtests/heat_integrationtests.conf
to override default configuration values. A local ConfigOpts is created
for each test to avoid any potential interaction with heat's
global CONF. Configuration options for credentials default to
being sourced from the environment.

The default tox testenv now excludes tests in heat_integrationtests.
A new testenv called "integration" will only run tests in
heat_integrationtests.

Integration tests will fail if preconditions are not met, including
a keystone endpoint, credentials and glance containing the expected
named image.

Devstack gate hooks have been moved to heat_integrationtests now that
the name of the package has been decided.

Change-Id: I174429c16bb606c5c325ee8b62c6e600ea77a6e6
Partial-Blueprint: functional-tests

heat_integrationtests/.gitignore

@@ -0,0 +1 @@
+heat_integrationtests.conf

heat_integrationtests/README.rst

@@ -0,0 +1,26 @@
+======================
+Heat integration tests
+======================
+
+These tests can be run against any heat-enabled OpenStack cloud, however
+defaults match running against a recent devstack.
+
+To run the tests against devstack, do the following:
+
+    # source devstack credentials
+    source /opt/stack/devstack/accrc/demo/demo
+    # run the heat integration tests with those credentials
+    cd /opt/stack/heat
+    tox -eintegration
+
+If custom configuration is required, copy the following file:
+
+    heat_integrationtests/heat_integrationtests.conf.sample
+
+to:
+
+    heat_integrationtests/heat_integrationtests.conf
+
+and make any required configuration changes before running:
+
+    tox -eintegration

heat_integrationtests/common/clients.py

@@ -0,0 +1,120 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import cinderclient.client
+import heatclient.client
+import keystoneclient.exceptions
+import keystoneclient.v2_0.client
+import neutronclient.v2_0.client
+import novaclient.client
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ClientManager(object):
+    """
+    Manager that provides access to the official python clients for
+    calling various OpenStack APIs.
+    """
+
+    CINDERCLIENT_VERSION = '1'
+    HEATCLIENT_VERSION = '1'
+    NOVACLIENT_VERSION = '2'
+
+    def __init__(self, conf):
+        self.conf = conf
+        self.identity_client = self._get_identity_client()
+        self.orchestration_client = self._get_orchestration_client()
+        self.compute_client = self._get_compute_client()
+        self.network_client = self._get_network_client()
+        self.volume_client = self._get_volume_client()
+
+    def _get_orchestration_client(self):
+        keystone = self._get_identity_client()
+        region = self.conf.region
+        token = keystone.auth_token
+        try:
+            endpoint = keystone.service_catalog.url_for(
+                attr='region',
+                filter_value=region,
+                service_type='orchestration',
+                endpoint_type='publicURL')
+        except keystoneclient.exceptions.EndpointNotFound:
+            return None
+        else:
+            return heatclient.client.Client(
+                self.HEATCLIENT_VERSION,
+                endpoint,
+                token=token,
+                username=self.conf.username,
+                password=self.conf.password)
+
+    def _get_identity_client(self):
+        return keystoneclient.v2_0.client.Client(
+            username=self.conf.username,
+            password=self.conf.password,
+            tenant_name=self.conf.tenant_name,
+            auth_url=self.conf.auth_url,
+            insecure=self.conf.disable_ssl_certificate_validation)
+
+    def _get_compute_client(self):
+
+        dscv = self.conf.disable_ssl_certificate_validation
+        region = self.conf.region
+
+        client_args = (
+            self.conf.username,
+            self.conf.password,
+            self.conf.tenant_name,
+            self.conf.auth_url
+        )
+
+        # Create our default Nova client to use in testing
+        return novaclient.client.Client(
+            self.NOVACLIENT_VERSION,
+            *client_args,
+            service_type='compute',
+            endpoint_type='publicURL',
+            region_name=region,
+            no_cache=True,
+            insecure=dscv,
+            http_log_debug=True)
+
+    def _get_network_client(self):
+        auth_url = self.conf.auth_url
+        dscv = self.conf.disable_ssl_certificate_validation
+
+        return neutronclient.v2_0.client.Client(
+            username=self.conf.username,
+            password=self.conf.password,
+            tenant_name=self.conf.tenant_name,
+            endpoint_type='publicURL',
+            auth_url=auth_url,
+            insecure=dscv)
+
+    def _get_volume_client(self):
+        auth_url = self.conf.auth_url
+        region = self.conf.region
+        endpoint_type = 'publicURL'
+        dscv = self.conf.disable_ssl_certificate_validation
+        return cinderclient.client.Client(
+            self.CINDERCLIENT_VERSION,
+            self.conf.username,
+            self.conf.password,
+            self.conf.tenant_name,
+            auth_url,
+            region_name=region,
+            endpoint_type=endpoint_type,
+            insecure=dscv,
+            http_log_debug=True)

heat_integrationtests/common/config.py

@@ -0,0 +1,114 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import sys
+
+from oslo.config import cfg
+
+import heat_integrationtests
+
+
+IntegrationTestGroup = [
+
+    cfg.StrOpt('username',
+               default=os.environ.get('OS_USERNAME'),
+               help="Username to use for API requests."),
+    cfg.StrOpt('password',
+               default=os.environ.get('OS_PASSWORD'),
+               help="API key to use when authenticating.",
+               secret=True),
+    cfg.StrOpt('tenant_name',
+               default=os.environ.get('OS_TENANT_NAME'),
+               help="Tenant name to use for API requests."),
+    cfg.StrOpt('auth_url',
+               default=os.environ.get('OS_AUTH_URL'),
+               help="Full URI of the OpenStack Identity API (Keystone), v2"),
+    cfg.StrOpt('region',
+               default=os.environ.get('OS_REGION_NAME'),
+               help="The region name to us"),
+    cfg.StrOpt('instance_type',
+               default='m1.micro',
+               help="Instance type for tests. Needs to be big enough for a "
+                    "full OS plus the test workload"),
+    cfg.StrOpt('image_ref',
+               default='Fedora-x86_64-20-20140618-sda',
+               help="Name of image to use for tests which boot servers."),
+    cfg.StrOpt('keypair_name',
+               default=None,
+               help="Name of existing keypair to launch servers with."),
+    cfg.StrOpt('minimal_image_ref',
+               default='cirros-0.3.2-x86_64-uec',
+               help="Name of minimal (e.g cirros) image to use when "
+                    "launching test instances."),
+    cfg.StrOpt('auth_version',
+               default='v2',
+               help="Identity API version to be used for authentication "
+                    "for API tests."),
+    cfg.BoolOpt('disable_ssl_certificate_validation',
+                default=False,
+                help="Set to True if using self-signed SSL certificates."),
+    cfg.IntOpt('build_interval',
+               default=4,
+               help="Time in seconds between build status checks."),
+    cfg.IntOpt('build_timeout',
+               default=1200,
+               help="Timeout in seconds to wait for a stack to build."),
+    cfg.StrOpt('network_for_ssh',
+               default='private',
+               help="Network used for SSH connections."),
+    cfg.StrOpt('fixed_network_name',
+               default='private',
+               help="Visible fixed network name "),
+    cfg.IntOpt('ssh_timeout',
+               default=300,
+               help="Timeout in seconds to wait for authentication to "
+                    "succeed."),
+    cfg.IntOpt('ip_version_for_ssh',
+               default=4,
+               help="IP version used for SSH connections."),
+    cfg.IntOpt('ssh_channel_timeout',
+               default=60,
+               help="Timeout in seconds to wait for output from ssh "
+                    "channel."),
+    cfg.IntOpt('tenant_network_mask_bits',
+               default=28,
+               help="The mask bits for tenant ipv4 subnets"),
+    cfg.IntOpt('volume_size',
+               default=1,
+               help='Default size in GB for volumes created by volumes tests'),
+]
+
+
+def init_conf(read_conf=True):
+
+    default_config_files = None
+    if read_conf:
+        confpath = os.path.join(
+            os.path.dirname(os.path.realpath(heat_integrationtests.__file__)),
+            'heat_integrationtests.conf')
+        if os.path.isfile(confpath):
+            default_config_files = [confpath]
+
+    conf = cfg.ConfigOpts()
+    conf(args=[], project='heat_integrationtests',
+         default_config_files=default_config_files)
+
+    for opt in IntegrationTestGroup:
+        conf.register_opt(opt)
+    return conf
+
+
+if __name__ == '__main__':
+    cfg.CONF = init_conf(False)
+    import heat.openstack.common.config.generator as generate
+    generate.generate(sys.argv[1:])

heat_integrationtests/common/exceptions.py

@@ -0,0 +1,79 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class IntegrationException(Exception):
+    """
+    Base Tempest Exception
+
+    To correctly use this class, inherit from it and define
+    a 'message' property. That message will get printf'd
+    with the keyword arguments provided to the constructor.
+    """
+    message = "An unknown exception occurred"
+
+    def __init__(self, *args, **kwargs):
+        super(IntegrationException, self).__init__()
+        try:
+            self._error_string = self.message % kwargs
+        except Exception:
+            # at least get the core message out if something happened
+            self._error_string = self.message
+        if len(args) > 0:
+            # If there is a non-kwarg parameter, assume it's the error
+            # message or reason description and tack it on to the end
+            # of the exception message
+            # Convert all arguments into their string representations...
+            args = ["%s" % arg for arg in args]
+            self._error_string = (self._error_string +
+                                  "\nDetails: %s" % '\n'.join(args))
+
+    def __str__(self):
+        return self._error_string
+
+
+class InvalidCredentials(IntegrationException):
+    message = "Invalid Credentials"
+
+
+class TimeoutException(IntegrationException):
+    message = "Request timed out"
+
+
+class BuildErrorException(IntegrationException):
+    message = "Server %(server_id)s failed to build and is in ERROR status"
+
+
+class StackBuildErrorException(IntegrationException):
+    message = ("Stack %(stack_identifier)s is in %(stack_status)s status "
+               "due to '%(stack_status_reason)s'")
+
+
+class StackResourceBuildErrorException(IntegrationException):
+    message = ("Resource %(resource_name)s in stack %(stack_identifier)s is "
+               "in %(resource_status)s status due to "
+               "'%(resource_status_reason)s'")
+
+
+class SSHTimeout(IntegrationException):
+    message = ("Connection to the %(host)s via SSH timed out.\n"
+               "User: %(user)s, Password: %(password)s")
+
+
+class SSHExecCommandFailed(IntegrationException):
+    """Raised when remotely executed command returns nonzero status."""
+    message = ("Command '%(command)s', exit status: %(exit_status)d, "
+               "Error:\n%(strerror)s")
+
+
+class ServerUnreachable(IntegrationException):
+    message = "The server is not reachable via the configured network"

heat_integrationtests/common/remote_client.py

@@ -0,0 +1,202 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import cStringIO
+import logging
+import paramiko
+import re
+import select
+import six
+import socket
+import time
+
+from heat_integrationtests.common import exceptions
+
+LOG = logging.getLogger(__name__)
+
+
+class Client(object):
+
+    def __init__(self, host, username, password=None, timeout=300, pkey=None,
+                 channel_timeout=10, look_for_keys=False, key_filename=None):
+        self.host = host
+        self.username = username
+        self.password = password
+        if isinstance(pkey, six.string_types):
+            pkey = paramiko.RSAKey.from_private_key(
+                cStringIO.StringIO(str(pkey)))
+        self.pkey = pkey
+        self.look_for_keys = look_for_keys
+        self.key_filename = key_filename
+        self.timeout = int(timeout)
+        self.channel_timeout = float(channel_timeout)
+        self.buf_size = 1024
+
+    def _get_ssh_connection(self, sleep=1.5, backoff=1):
+        """Returns an ssh connection to the specified host."""
+        bsleep = sleep
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(
+            paramiko.AutoAddPolicy())
+        _start_time = time.time()
+        if self.pkey is not None:
+            LOG.info("Creating ssh connection to '%s' as '%s'"
+                     " with public key authentication",
+                     self.host, self.username)
+        else:
+            LOG.info("Creating ssh connection to '%s' as '%s'"
+                     " with password %s",
+                     self.host, self.username, str(self.password))
+        attempts = 0
+        while True:
+            try:
+                ssh.connect(self.host, username=self.username,
+                            password=self.password,
+                            look_for_keys=self.look_for_keys,
+                            key_filename=self.key_filename,
+                            timeout=self.channel_timeout, pkey=self.pkey)
+                LOG.info("ssh connection to %s@%s successfuly created",
+                         self.username, self.host)
+                return ssh
+            except (socket.error,
+                    paramiko.SSHException) as e:
+                if self._is_timed_out(_start_time):
+                    LOG.exception("Failed to establish authenticated ssh"
+                                  " connection to %s@%s after %d attempts",
+                                  self.username, self.host, attempts)
+                    raise exceptions.SSHTimeout(host=self.host,
+                                                user=self.username,
+                                                password=self.password)
+                bsleep += backoff
+                attempts += 1
+                LOG.warning("Failed to establish authenticated ssh"
+                            " connection to %s@%s (%s). Number attempts: %s."
+                            " Retry after %d seconds.",
+                            self.username, self.host, e, attempts, bsleep)
+                time.sleep(bsleep)
+
+    def _is_timed_out(self, start_time):
+        return (time.time() - self.timeout) > start_time
+
+    def exec_command(self, cmd):
+        """
+        Execute the specified command on the server.
+
+        Note that this method is reading whole command outputs to memory, thus
+        shouldn't be used for large outputs.
+
+        :returns: data read from standard output of the command.
+        :raises: SSHExecCommandFailed if command returns nonzero
+                 status. The exception contains command status stderr content.
+        """
+        ssh = self._get_ssh_connection()
+        transport = ssh.get_transport()
+        channel = transport.open_session()
+        channel.fileno()  # Register event pipe
+        channel.exec_command(cmd)
+        channel.shutdown_write()
+        out_data = []
+        err_data = []
+        poll = select.poll()
+        poll.register(channel, select.POLLIN)
+        start_time = time.time()
+
+        while True:
+            ready = poll.poll(self.channel_timeout)
+            if not any(ready):
+                if not self._is_timed_out(start_time):
+                    continue
+                raise exceptions.TimeoutException(
+                    "Command: '{0}' executed on host '{1}'.".format(
+                        cmd, self.host))
+            if not ready[0]:  # If there is nothing to read.
+                continue
+            out_chunk = err_chunk = None
+            if channel.recv_ready():
+                out_chunk = channel.recv(self.buf_size)
+                out_data += out_chunk,
+            if channel.recv_stderr_ready():
+                err_chunk = channel.recv_stderr(self.buf_size)
+                err_data += err_chunk,
+            if channel.closed and not err_chunk and not out_chunk:
+                break
+        exit_status = channel.recv_exit_status()
+        if 0 != exit_status:
+            raise exceptions.SSHExecCommandFailed(
+                command=cmd, exit_status=exit_status,
+                strerror=''.join(err_data))
+        return ''.join(out_data)
+
+    def test_connection_auth(self):
+        """Raises an exception when we can not connect to server via ssh."""
+        connection = self._get_ssh_connection()
+        connection.close()
+
+
+class RemoteClient():
+
+    # NOTE(afazekas): It should always get an address instead of server
+    def __init__(self, server, username, password=None, pkey=None,
+                 conf=None):
+        self.conf = conf
+        ssh_timeout = self.conf.ssh_timeout
+        network = self.conf.network_for_ssh
+        ip_version = self.conf.ip_version_for_ssh
+        ssh_channel_timeout = self.conf.ssh_channel_timeout
+        if isinstance(server, six.string_types):
+            ip_address = server
+        else:
+            addresses = server['addresses'][network]
+            for address in addresses:
+                if address['version'] == ip_version:
+                    ip_address = address['addr']
+                    break
+            else:
+                raise exceptions.ServerUnreachable()
+        self.ssh_client = Client(ip_address, username, password,
+                                 ssh_timeout, pkey=pkey,
+                                 channel_timeout=ssh_channel_timeout)
+
+    def exec_command(self, cmd):
+        return self.ssh_client.exec_command(cmd)
+
+    def validate_authentication(self):
+        """Validate ssh connection and authentication
+           This method raises an Exception when the validation fails.
+        """
+        self.ssh_client.test_connection_auth()
+
+    def get_partitions(self):
+        # Return the contents of /proc/partitions
+        command = 'cat /proc/partitions'
+        output = self.exec_command(command)
+        return output
+
+    def get_boot_time(self):
+        cmd = 'cut -f1 -d. /proc/uptime'
+        boot_secs = self.exec_command(cmd)
+        boot_time = time.time() - int(boot_secs)
+        return time.localtime(boot_time)
+
+    def write_to_console(self, message):
+        message = re.sub("([$\\`])", "\\\\\\\\\\1", message)
+        # usually to /dev/ttyS0
+        cmd = 'sudo sh -c "echo \\"%s\\" >/dev/console"' % message
+        return self.exec_command(cmd)
+
+    def ping_host(self, host):
+        cmd = 'ping -c1 -w1 %s' % host
+        return self.exec_command(cmd)
+
+    def get_ip_list(self):
+        cmd = "/bin/ip address"
+        return self.exec_command(cmd)

heat_integrationtests/common/test.py

@@ -0,0 +1,306 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import os
+import random
+import re
+import six
+import subprocess
+import testtools
+import time
+
+from heatclient import exc as heat_exceptions
+
+from heat.openstack.common import timeutils
+from heat_integrationtests.common import clients
+from heat_integrationtests.common import config
+from heat_integrationtests.common import exceptions
+from heat_integrationtests.common import remote_client
+
+LOG = logging.getLogger(__name__)
+
+
+def call_until_true(func, duration, sleep_for):
+    """
+    Call the given function until it returns True (and return True) or
+    until the specified duration (in seconds) elapses (and return
+    False).
+
+    :param func: A zero argument callable that returns True on success.
+    :param duration: The number of seconds for which to attempt a
+        successful call of the function.
+    :param sleep_for: The number of seconds to sleep after an unsuccessful
+                      invocation of the function.
+    """
+    now = time.time()
+    timeout = now + duration
+    while now < timeout:
+        if func():
+            return True
+        LOG.debug("Sleeping for %d seconds", sleep_for)
+        time.sleep(sleep_for)
+        now = time.time()
+    return False
+
+
+def rand_name(name=''):
+    randbits = str(random.randint(1, 0x7fffffff))
+    if name:
+        return name + '-' + randbits
+    else:
+        return randbits
+
+
+class HeatIntegrationTest(testtools.TestCase):
+
+    def setUp(self):
+        super(HeatIntegrationTest, self).setUp()
+
+        self.conf = config.init_conf()
+
+        self.assertIsNotNone(self.conf.auth_url,
+                             'No auth_url configured')
+        self.assertIsNotNone(self.conf.username,
+                             'No username configured')
+        self.assertIsNotNone(self.conf.password,
+                             'No password configured')
+
+        self.manager = clients.ClientManager(self.conf)
+        self.identity_client = self.manager.identity_client
+        self.orchestration_client = self.manager.orchestration_client
+        self.compute_client = self.manager.compute_client
+        self.network_client = self.manager.network_client
+        self.volume_client = self.manager.volume_client
+
+    def status_timeout(self, things, thing_id, expected_status,
+                       error_status='ERROR',
+                       not_found_exception=heat_exceptions.NotFound):
+        """
+        Given a thing and an expected status, do a loop, sleeping
+        for a configurable amount of time, checking for the
+        expected status to show. At any time, if the returned
+        status of the thing is ERROR, fail out.
+        """
+        self._status_timeout(things, thing_id,
+                             expected_status=expected_status,
+                             error_status=error_status,
+                             not_found_exception=not_found_exception)
+
+    def _status_timeout(self,
+                        things,
+                        thing_id,
+                        expected_status=None,
+                        allow_notfound=False,
+                        error_status='ERROR',
+                        not_found_exception=heat_exceptions.NotFound):
+
+        log_status = expected_status if expected_status else ''
+        if allow_notfound:
+            log_status += ' or NotFound' if log_status != '' else 'NotFound'
+
+        def check_status():
+            # python-novaclient has resources available to its client
+            # that all implement a get() method taking an identifier
+            # for the singular resource to retrieve.
+            try:
+                thing = things.get(thing_id)
+            except not_found_exception:
+                if allow_notfound:
+                    return True
+                raise
+            except Exception as e:
+                if allow_notfound and self.not_found_exception(e):
+                    return True
+                raise
+
+            new_status = thing.status
+
+            # Some components are reporting error status in lower case
+            # so case sensitive comparisons can really mess things
+            # up.
+            if new_status.lower() == error_status.lower():
+                message = ("%s failed to get to expected status (%s). "
+                           "In %s state.") % (thing, expected_status,
+                                              new_status)
+                raise exceptions.BuildErrorException(message,
+                                                     server_id=thing_id)
+            elif new_status == expected_status and expected_status is not None:
+                return True  # All good.
+            LOG.debug("Waiting for %s to get to %s status. "
+                      "Currently in %s status",
+                      thing, log_status, new_status)
+        if not call_until_true(
+                check_status,
+                self.conf.build_timeout,
+                self.conf.build_interval):
+            message = ("Timed out waiting for thing %s "
+                       "to become %s") % (thing_id, log_status)
+            raise exceptions.TimeoutException(message)
+
+    def get_remote_client(self, server_or_ip, username, private_key=None):
+        if isinstance(server_or_ip, six.string_types):
+            ip = server_or_ip
+        else:
+            network_name_for_ssh = self.conf.network_for_ssh
+            ip = server_or_ip.networks[network_name_for_ssh][0]
+        if private_key is None:
+            private_key = self.keypair.private_key
+        linux_client = remote_client.RemoteClient(ip, username,
+                                                  pkey=private_key,
+                                                  conf=self.conf)
+        try:
+            linux_client.validate_authentication()
+        except exceptions.SSHTimeout:
+            LOG.exception('ssh connection to %s failed' % ip)
+            raise
+
+        return linux_client
+
+    def _log_console_output(self, servers=None):
+        if not servers:
+            servers = self.compute_client.servers.list()
+        for server in servers:
+            LOG.debug('Console output for %s', server.id)
+            LOG.debug(server.get_console_output())
+
+    def _load_template(self, base_file, file_name):
+        filepath = os.path.join(os.path.dirname(os.path.realpath(base_file)),
+                                file_name)
+        with open(filepath) as f:
+            return f.read()
+
+    def create_keypair(self, client=None, name=None):
+        if client is None:
+            client = self.compute_client
+        if name is None:
+            name = rand_name('heat-keypair')
+        keypair = client.keypairs.create(name)
+        self.assertEqual(keypair.name, name)
+
+        def delete_keypair():
+            keypair.delete()
+
+        self.addCleanup(delete_keypair)
+        return keypair
+
+    @classmethod
+    def _stack_rand_name(cls):
+        return rand_name(cls.__name__)
+
+    def _get_default_network(self):
+        networks = self.network_client.list_networks()
+        for net in networks['networks']:
+            if net['name'] == self.conf.fixed_network_name:
+                return net
+
+    @staticmethod
+    def _stack_output(stack, output_key):
+        """Return a stack output value for a given key."""
+        return next((o['output_value'] for o in stack.outputs
+                    if o['output_key'] == output_key), None)
+
+    def _ping_ip_address(self, ip_address, should_succeed=True):
+        cmd = ['ping', '-c1', '-w1', ip_address]
+
+        def ping():
+            proc = subprocess.Popen(cmd,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+            proc.wait()
+            return (proc.returncode == 0) == should_succeed
+
+        return call_until_true(
+            ping, self.conf.build_timeout, 1)
+
+    def _wait_for_resource_status(self, stack_identifier, resource_name,
+                                  status, failure_pattern='^.*_FAILED$',
+                                  success_on_not_found=False):
+        """Waits for a Resource to reach a given status."""
+        fail_regexp = re.compile(failure_pattern)
+        build_timeout = self.conf.build_timeout
+        build_interval = self.conf.build_interval
+
+        start = timeutils.utcnow()
+        while timeutils.delta_seconds(start,
+                                      timeutils.utcnow()) < build_timeout:
+            try:
+                res = self.client.resources.get(
+                    stack_identifier, resource_name)
+            except heat_exceptions.HTTPNotFound:
+                if success_on_not_found:
+                    return
+                # ignore this, as the resource may not have
+                # been created yet
+            else:
+                if res.resource_status == status:
+                    return
+                if fail_regexp.search(res.resource_status):
+                    raise exceptions.StackResourceBuildErrorException(
+                        resource_name=res.resource_name,
+                        stack_identifier=stack_identifier,
+                        resource_status=res.resource_status,
+                        resource_status_reason=res.resource_status_reason)
+            time.sleep(build_interval)
+
+        message = ('Resource %s failed to reach %s status within '
+                   'the required time (%s s).' %
+                   (res.resource_name, status, build_timeout))
+        raise exceptions.TimeoutException(message)
+
+    def _wait_for_stack_status(self, stack_identifier, status,
+                               failure_pattern='^.*_FAILED$',
+                               success_on_not_found=False):
+        """
+        Waits for a Stack to reach a given status.
+
+        Note this compares the full $action_$status, e.g
+        CREATE_COMPLETE, not just COMPLETE which is exposed
+        via the status property of Stack in heatclient
+        """
+        fail_regexp = re.compile(failure_pattern)
+        build_timeout = self.conf.build_timeout
+        build_interval = self.conf.build_interval
+
+        start = timeutils.utcnow()
+        while timeutils.delta_seconds(start,
+                                      timeutils.utcnow()) < build_timeout:
+            try:
+                stack = self.client.stacks.get(stack_identifier)
+            except heat_exceptions.HTTPNotFound:
+                if success_on_not_found:
+                    return
+                # ignore this, as the resource may not have
+                # been created yet
+            else:
+                if stack.stack_status == status:
+                    return
+                if fail_regexp.search(stack.stack_status):
+                    raise exceptions.StackBuildErrorException(
+                        stack_identifier=stack_identifier,
+                        stack_status=stack.stack_status,
+                        stack_status_reason=stack.stack_status_reason)
+            time.sleep(build_interval)
+
+        message = ('Stack %s failed to reach %s status within '
+                   'the required time (%s s).' %
+                   (stack.stack_name, status, build_timeout))
+        raise exceptions.TimeoutException(message)
+
+    def _stack_delete(self, stack_identifier):
+        try:
+            self.client.stacks.delete(stack_identifier)
+        except heat_exceptions.HTTPNotFound:
+            pass
+        self._wait_for_stack_status(
+            stack_identifier, 'DELETE_COMPLETE',
+            success_on_not_found=True)

heat_integrationtests/generate_sample.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# generates sample configuration file heat_integrationtests.conf.sample
+unset OS_USERNAME
+unset OS_PASSWORD
+unset OS_TENANT_NAME
+unset OS_AUTH_URL
+unset OS_REGION_NAME
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+pushd $DIR/..
+PYTHONPATH=. python -m heat_integrationtests.common.config \
+          heat_integrationtests/common/config.py \
+          > heat_integrationtests/heat_integrationtests.conf.sample
+popd

heat_integrationtests/heat_integrationtests.conf.sample

@@ -0,0 +1,78 @@
+[DEFAULT]
+
+#
+# Options defined in heat_integrationtests.common.config
+#
+
+# Username to use for API requests. (string value)
+#username=<None>
+
+# API key to use when authenticating. (string value)
+#password=<None>
+
+# Tenant name to use for API requests. (string value)
+#tenant_name=<None>
+
+# Full URI of the OpenStack Identity API (Keystone), v2
+# (string value)
+#auth_url=<None>
+
+# The region name to us (string value)
+#region=<None>
+
+# Instance type for tests. Needs to be big enough for a full
+# OS plus the test workload (string value)
+#instance_type=m1.micro
+
+# Name of image to use for tests which boot servers. (string
+# value)
+#image_ref=Fedora-x86_64-20-20140618-sda
+
+# Name of existing keypair to launch servers with. (string
+# value)
+#keypair_name=<None>
+
+# Name of minimal (e.g cirros) image to use when launching
+# test instances. (string value)
+#minimal_image_ref=cirros-0.3.2-x86_64-uec
+
+# Identity API version to be used for authentication for API
+# tests. (string value)
+#auth_version=v2
+
+# Set to True if using self-signed SSL certificates. (boolean
+# value)
+#disable_ssl_certificate_validation=false
+
+# Time in seconds between build status checks. (integer value)
+#build_interval=4
+
+# Timeout in seconds to wait for a stack to build. (integer
+# value)
+#build_timeout=1200
+
+# Network used for SSH connections. (string value)
+#network_for_ssh=private
+
+# Visible fixed network name  (string value)
+#fixed_network_name=private
+
+# Timeout in seconds to wait for authentication to succeed.
+# (integer value)
+#ssh_timeout=300
+
+# IP version used for SSH connections. (integer value)
+#ip_version_for_ssh=4
+
+# Timeout in seconds to wait for output from ssh channel.
+# (integer value)
+#ssh_channel_timeout=60
+
+# The mask bits for tenant ipv4 subnets (integer value)
+#tenant_network_mask_bits=28
+
+# Default size in GB for volumes created by volumes tests
+# (integer value)
+#volume_size=1
+
+

heat_integrationtests/post_test_hook.sh

@@ -14,5 +14,6 @@
 
 # This script is executed inside post_test_hook function in devstack gate.
 
+source /opt/stack/new/devstack/accrc/demo/demo
 cd /opt/stack/new/heat
-sudo tox -efunctional
+sudo tox -eintegration

test-requirements.txt

@@ -12,6 +12,7 @@ mox>=0.5.3
 MySQL-python
 oslosphinx>=2.2.0  # Apache-2.0
 oslotest>=1.1.0  # Apache-2.0
+paramiko>=1.13.0
 psycopg2
 sphinx>=1.1.2,!=1.2.0,<1.3
 testrepository>=0.0.18

tox.ini

@@ -13,17 +13,17 @@ install_command = pip install {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-  python setup.py testr --slowest --testr-args='^(?!functionaltests) {posargs}'
+  python setup.py testr --slowest --testr-args='^(?!heat_integrationtests) {posargs}'
 
 whitelist_externals = bash
 
-[testenv:functional]
+[testenv:integration]
 commands =
-  python -c "print('TODO: functional tests')"
+  python setup.py testr --slowest --testr-args='--concurrency=1 ^heat_integrationtests {posargs}'
 
 [testenv:pep8]
 commands =
-    flake8 heat bin/heat-api bin/heat-api-cfn bin/heat-api-cloudwatch bin/heat-engine bin/heat-manage contrib functionaltests
+    flake8 heat bin/heat-api bin/heat-api-cfn bin/heat-api-cloudwatch bin/heat-engine bin/heat-manage contrib heat_integrationtests
     {toxinidir}/tools/config/check_uptodate.sh
     # Check that .po and .pot files are valid:
     bash -c "find heat -type f -regex '.*\.pot?' -print0|xargs -0 -n 1 msgfmt --check-format -o /dev/null"