Reorganize test module structure

Moved test modules:
 tests/             ->  tests/unit
 test_ci/           ->  tests/ci
 tests_functional   ->  tests/functional
 rally/hacking      ->  tests/hacking

Add testing read me file

Co-Authored-By: Boris Pavlovic <boris@pavlovic.me>
Co-Authored-By: Andrey Kurilin <akurilin@mirantis.com>

Change-Id: I57c09d892da4adf863c358a4d63e3543b50d10b7

.testr.conf

@@ -1,4 +1,4 @@
 [DEFAULT]
-test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests/unit $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list

doc/source/index.rst

@@ -42,6 +42,7 @@ Deeper in Rally:
    verify
    installation
    usage
+   testing
    feature_requests
    user_stories
 

doc/source/testing.rst

@@ -0,0 +1 @@
+../../tests/README.rst

doc/source/user_stories.rst

@@ -1,6 +1,6 @@
 
-User stories:
-=============
+User stories
+============
 
 .. toctree::
    :glob:

tests/README.rst

@@ -0,0 +1,75 @@
+Testing
+=======
+
+Please, don't hesitate to write tests ;)
+
+
+Unit tests
+----------
+
+*Files: /tests/unit/**
+
+The goal of unit tests is to ensure that internal parts of the code work properly.
+All internal methods should be fully covered by unit tests with a reasonable mocks usage.
+
+
+About Rally unit tests:
+
+- All `unit tests <http://en.wikipedia.org/wiki/Unit_testing>`_ are located inside /tests/unit/*
+- Tests are written on top of: *testtools*, *fixtures* and *mock* libs
+- `Tox <https://tox.readthedocs.org/en/latest/>`_ is used to run unit tests
+
+
+To run unit tests locally::
+
+  $ pip install tox
+  $ tox
+
+To run py26, py27 or pep8 only::
+
+  $ tox -e <name>
+
+  #NOTE: <name> is one of py26, py27 or pep8
+
+To get test coverage::
+
+  $ tox -e cover
+
+  #NOTE: Results will be in /cover/index.html
+
+To generate docs::
+
+  $ tox -e docs
+
+  #NOTE: Documentation will be in doc/source/_build/html/index.html
+
+Functional tests
+----------------
+
+*Files: /tests/functional/**
+
+The goal of `functional tests <https://en.wikipedia.org/wiki/Functional_testing>`_ is to check that everything works well together.
+Fuctional tests use Rally API only and check responses without touching internal parts.
+
+To run functional tests locally::
+
+  $ source openrc
+  $ rally deployment create --from-env --name testing
+  $ tox -e cli
+
+  #NOTE: openrc file with OpenStack admin credentials
+
+Rally CI scripts
+----------------
+
+*Files: /tests/ci/**
+
+This directory contains scripts and files related to the Rally CI system.
+
+Rally Style Commandments
+------------------------
+
+*File: /tests/hacking/checks.py*
+
+This module contains Rally specific hacking rules for checking commandments.
+

tests/ci/rally-gate.sh

@@ -45,7 +45,7 @@ rally show keypairs
 rally -v task start --task $SCENARIO
 
 mkdir -p rally-plot/extra
-cp $BASE/new/rally/tests_ci/rally-gate/index.html rally-plot/extra/index.html
+cp $BASE/new/rally/tests/ci/rally-gate/index.html rally-plot/extra/index.html
 cp $SCENARIO rally-plot/task.txt
 tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR .
 rally task plot2html --out rally-plot/results.html

tests/ci/rally-integrated.sh

@@ -3,7 +3,7 @@
 env
 
 mkdir -p .testrepository
-python -m subunit.run discover tests_functional > .testrepository/subunit.log
+python -m subunit.run discover tests/functional > .testrepository/subunit.log
 EXIT_CODE=$?
 
 subunit2pyunit < .testrepository/subunit.log

tests/hacking/README.rst

@@ -1,5 +1,5 @@
-Nova Style Commandments
-=======================
+Rally Style Commandments
+========================
 
 - Step 1: Read the OpenStack Style Commandments
   http://docs.openstack.org/developer/hacking/

tests/unit/aas/rest/base.py

@@ -21,7 +21,7 @@ import pecan
 import pecan.testing
 from requests import utils
 
-from tests import test
+from tests.unit import test
 
 PATH_PREFIX = '/v1'
 

tests/unit/aas/rest/controllers/test_root.py

@@ -15,7 +15,7 @@
 
 from __future__ import print_function
 
-from tests.aas.rest import base
+from tests.unit.aas.rest import base
 
 
 class TestRoot(base.PecanControllerTest):

tests/unit/aas/rest/test_types.py

@@ -18,7 +18,7 @@
 from __future__ import print_function
 
 from rally.aas.rest import types
-from tests import test
+from tests.unit import test
 
 
 class TestLink(test.TestCase):

tests/unit/benchmark/context/cleanup/test_admin_cleanup.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.context.cleanup import admin_cleanup
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 BASE = "rally.benchmark.context.cleanup.admin_cleanup"

tests/unit/benchmark/context/cleanup/test_user_cleanup.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.context.cleanup import user_cleanup
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 BASE = "rally.benchmark.context.cleanup.user_cleanup"

tests/unit/benchmark/context/cleanup/test_utils.py

@@ -18,8 +18,8 @@ import mock
 
 from rally.benchmark.context.cleanup import utils
 from rally.benchmark import scenarios
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class CleanupUtilsTestCase(test.TestCase):

tests/unit/benchmark/context/quotas/test_cinder_quotas.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.context.quotas import cinder_quotas
-from tests import test
+from tests.unit import test
 
 
 class CinderQuotasTestCase(test.TestCase):

tests/unit/benchmark/context/quotas/test_designate_quotas.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.context.quotas import designate_quotas
-from tests import test
+from tests.unit import test
 
 
 class DesignateQuotasTestCase(test.TestCase):

tests/unit/benchmark/context/quotas/test_neutron_quotas.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.context.quotas import neutron_quotas as quotas
-from tests import test
+from tests.unit import test
 
 
 class NeutronQuotasTestCase(test.TestCase):

tests/unit/benchmark/context/quotas/test_nova_quotas.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.context.quotas import nova_quotas as quotas
-from tests import test
+from tests.unit import test
 
 
 class NovaQuotasTestCase(test.TestCase):

tests/unit/benchmark/context/quotas/test_quotas.py

@@ -20,7 +20,7 @@ import jsonschema
 import mock
 
 from rally.benchmark.context.quotas import quotas
-from tests import test
+from tests.unit import test
 
 
 class QuotasTestCase(test.TestCase):

tests/unit/benchmark/context/sahara/test_sahara_cluster.py

@@ -16,7 +16,7 @@ import mock
 from oslo.config import cfg
 
 from rally.benchmark.context.sahara import sahara_cluster
-from tests import test
+from tests.unit import test
 
 CONF = cfg.CONF
 

tests/unit/benchmark/context/sahara/test_sahara_edp.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.context.sahara import sahara_edp
-from tests import test
+from tests.unit import test
 
 BASE_CTX = "rally.benchmark.context"
 CTX = "rally.benchmark.context.sahara"

tests/unit/benchmark/context/sahara/test_sahara_image.py

@@ -16,7 +16,7 @@ import mock
 
 from rally.benchmark.context.sahara import sahara_image
 from rally import exceptions
-from tests import test
+from tests.unit import test
 
 BASE_CTX = "rally.benchmark.context"
 CTX = "rally.benchmark.context.sahara"

tests/unit/benchmark/context/test_base.py

@@ -20,8 +20,8 @@ import mock
 from rally.benchmark.context import base
 from rally import exceptions
 from rally import utils
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class BaseContextTestCase(test.TestCase):

tests/unit/benchmark/context/test_images.py

@@ -20,8 +20,8 @@ import mock
 
 from rally.benchmark.context import images
 from rally import exceptions
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 CTX = "rally.benchmark.context"
 SCN = "rally.benchmark.scenarios"

tests/unit/benchmark/context/test_keypair.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.context import keypair
-from tests import test
+from tests.unit import test
 
 CTX = "rally.benchmark.context"
 

tests/unit/benchmark/context/test_roles.py

@@ -17,8 +17,8 @@ import mock
 
 from rally.benchmark.context import roles
 from rally import exceptions
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class RoleGeneratorTestCase(test.TestCase):

tests/unit/benchmark/context/test_secgroups.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.context import secgroup
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class SecGroupContextTestCase(test.TestCase):

tests/unit/benchmark/context/test_tempest.py

@@ -19,7 +19,7 @@ import mock
 
 from rally.benchmark.context import tempest
 from rally import exceptions
-from tests import test
+from tests.unit import test
 
 
 CONTEXT = "rally.benchmark.context.tempest"

tests/unit/benchmark/context/test_users.py

@@ -19,7 +19,7 @@ import mock
 
 from rally.benchmark.context import users
 from rally.benchmark import utils
-from tests import test
+from tests.unit import test
 
 
 run_concurrent = (lambda dummy, cls, f, args: list(

tests/unit/benchmark/context/test_volumes.py

@@ -18,8 +18,8 @@ import copy
 import mock
 
 from rally.benchmark.context import volumes
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 CTX = "rally.benchmark.context"
 SCN = "rally.benchmark.scenarios"

tests/unit/benchmark/processing/test_plot.py

@@ -18,7 +18,7 @@ import json
 import mock
 
 from rally.benchmark.processing import plot
-from tests import test
+from tests.unit import test
 
 
 class PlotTestCase(test.TestCase):

tests/unit/benchmark/processing/test_utils.py

@@ -15,7 +15,7 @@
 
 from rally.benchmark.processing import utils
 from rally import exceptions
-from tests import test
+from tests.unit import test
 
 
 class MathTestCase(test.TestCase):

tests/unit/benchmark/runners/test_base.py

@@ -20,8 +20,8 @@ from rally.benchmark.runners import base
 from rally.benchmark.runners import serial
 from rally.benchmark.scenarios import base as scenario_base
 from rally import exceptions
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class ScenarioHelpersTestCase(test.TestCase):

tests/unit/benchmark/runners/test_constant.py

@@ -18,8 +18,8 @@ import jsonschema
 from rally.benchmark.runners import base
 from rally.benchmark.runners import constant
 from rally import consts
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class ConstantScenarioRunnerTestCase(test.TestCase):

tests/unit/benchmark/runners/test_rps.py

@@ -19,8 +19,8 @@ import mock
 from rally.benchmark.runners import base
 from rally.benchmark.runners import rps
 from rally import consts
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class RPSScenarioRunnerTestCase(test.TestCase):

tests/unit/benchmark/runners/test_serial.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.runners import serial
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class SerialScenarioRunnerTestCase(test.TestCase):

tests/unit/benchmark/scenarios/authenticate/test_authenticate.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.scenarios.authenticate import authenticate
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class AuthenticateTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_alarms.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.scenarios.ceilometer import alarms
-from tests import test
+from tests.unit import test
 
 
 class CeilometerAlarmsTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_meters.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.scenarios.ceilometer import meters
-from tests import test
+from tests.unit import test
 
 
 class CeilometerMetersTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_queries.py

@@ -17,7 +17,7 @@ import json
 import mock
 
 from rally.benchmark.scenarios.ceilometer import queries
-from tests import test
+from tests.unit import test
 
 
 class CeilometerQueriesTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_resources.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.scenarios.ceilometer import resources
-from tests import test
+from tests.unit import test
 
 
 class CeilometerResourcesTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_stats.py

@@ -15,7 +15,7 @@
 import mock
 
 from rally.benchmark.scenarios.ceilometer import stats
-from tests import test
+from tests.unit import test
 
 
 class CeilometerStatsTestCase(test.TestCase):

tests/unit/benchmark/scenarios/ceilometer/test_utils.py

@@ -15,8 +15,8 @@
 import mock
 
 from rally.benchmark.scenarios.ceilometer import utils
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 UTILS = "rally.benchmark.scenarios.ceilometer.utils"
 

tests/unit/benchmark/scenarios/cinder/test_utils.py

@@ -18,7 +18,7 @@ from oslo.config import cfg
 from oslotest import mockpatch
 
 from rally.benchmark.scenarios.cinder import utils
-from tests import test
+from tests.unit import test
 
 BM_UTILS = 'rally.benchmark.utils'
 CINDER_UTILS = "rally.benchmark.scenarios.cinder.utils"

tests/unit/benchmark/scenarios/cinder/test_volumes.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.cinder import volumes
-from tests import test
+from tests.unit import test
 
 
 CINDER_VOLUMES = "rally.benchmark.scenarios.cinder.volumes.CinderVolumes"

tests/unit/benchmark/scenarios/designate/test_basic.py

@@ -17,7 +17,7 @@
 import mock
 
 from rally.benchmark.scenarios.designate import basic
-from tests import test
+from tests.unit import test
 
 DESIGNATE_BASIC = "rally.benchmark.scenarios.designate.basic.DesignateBasic"
 

tests/unit/benchmark/scenarios/designate/test_utils.py

@@ -17,7 +17,7 @@
 import mock
 
 from rally.benchmark.scenarios.designate import utils
-from tests import test
+from tests.unit import test
 
 
 DESIGNATE_UTILS = "rally.benchmark.scenarios.designate.utils."

tests/unit/benchmark/scenarios/dummy/test_dummy.py

@@ -15,7 +15,7 @@ import mock
 
 from rally.benchmark.scenarios.dummy import dummy
 from rally import exceptions
-from tests import test
+from tests.unit import test
 
 
 class DummyTestCase(test.TestCase):

tests/unit/benchmark/scenarios/glance/test_images.py

@@ -19,8 +19,8 @@ from rally.benchmark.scenarios.glance import images
 from rally.benchmark.scenarios.nova import servers
 from rally.objects import endpoint
 from rally import osclients
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 GLANCE_IMAGES = "rally.benchmark.scenarios.glance.images.GlanceImages"
 

tests/unit/benchmark/scenarios/glance/test_utils.py

@@ -20,8 +20,8 @@ from oslotest import mockpatch
 from rally.benchmark.scenarios.glance import utils
 from rally.benchmark import utils as butils
 from rally import exceptions as rally_exceptions
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 BM_UTILS = 'rally.benchmark.utils'
 GLANCE_UTILS = 'rally.benchmark.scenarios.glance.utils'

tests/unit/benchmark/scenarios/heat/test_stacks.py

@@ -18,7 +18,7 @@ import tempfile
 import mock
 
 from rally.benchmark.scenarios.heat import stacks
-from tests import test
+from tests.unit import test
 
 HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks"
 

tests/unit/benchmark/scenarios/heat/test_utils.py

@@ -17,7 +17,7 @@ import mock
 from oslotest import mockpatch
 
 from rally.benchmark.scenarios.heat import utils
-from tests import test
+from tests.unit import test
 
 BM_UTILS = 'rally.benchmark.utils'
 HEAT_UTILS = 'rally.benchmark.scenarios.heat.utils'

tests/unit/benchmark/scenarios/keystone/test_basic.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.keystone import basic
-from tests import test
+from tests.unit import test
 
 BASE = "rally.benchmark.scenarios.keystone."
 BASIC = BASE + "basic.KeystoneBasic."

tests/unit/benchmark/scenarios/keystone/test_utils.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.scenarios.keystone import utils
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 UTILS = "rally.benchmark.scenarios.keystone.utils."
 

tests/unit/benchmark/scenarios/neutron/test_network.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.neutron import network
-from tests import test
+from tests.unit import test
 
 NEUTRON_NETWORKS = "rally.benchmark.scenarios.neutron.network.NeutronNetworks"
 

tests/unit/benchmark/scenarios/neutron/test_utils.py

@@ -17,8 +17,8 @@ import mock
 import netaddr
 
 from rally.benchmark.scenarios.neutron import utils
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 NEUTRON_UTILS = "rally.benchmark.scenarios.neutron.utils."

tests/unit/benchmark/scenarios/nova/test_servers.py

@@ -19,8 +19,8 @@ from rally.benchmark.scenarios.nova import servers
 from rally import exceptions as rally_exceptions
 from rally.objects import endpoint
 from rally import osclients
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 NOVA_SERVERS_MODULE = "rally.benchmark.scenarios.nova.servers"

tests/unit/benchmark/scenarios/nova/test_utils.py

@@ -20,8 +20,8 @@ from oslotest import mockpatch
 from rally.benchmark.scenarios.nova import utils
 from rally.benchmark import utils as butils
 from rally import exceptions as rally_exceptions
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 BM_UTILS = 'rally.benchmark.utils'
 NOVA_UTILS = "rally.benchmark.scenarios.nova.utils"

tests/unit/benchmark/scenarios/quotas/test_quotas.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.quotas import quotas
-from tests import test
+from tests.unit import test
 
 
 class QuotasTestCase(test.TestCase):

tests/unit/benchmark/scenarios/quotas/test_utils.py

@@ -16,8 +16,8 @@
 import mock
 
 from rally.benchmark.scenarios.quotas import utils
-from tests import fakes
-from tests import test
+from tests.unit import fakes
+from tests.unit import test
 
 
 class QuotasScenarioTestCase(test.TestCase):

tests/unit/benchmark/scenarios/requests/test_http_requests.py

@@ -14,7 +14,7 @@
 import mock
 
 from rally.benchmark.scenarios.requests import http_requests
-from tests import test
+from tests.unit import test
 
 SCN = "rally.benchmark.scenarios"
 

tests/unit/benchmark/scenarios/sahara/test_clusters.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.sahara import clusters
-from tests import test
+from tests.unit import test
 
 SAHARA_CLUSTERS = "rally.benchmark.scenarios.sahara.clusters.SaharaClusters"
 SAHARA_UTILS = 'rally.benchmark.scenarios.sahara.utils'

tests/unit/benchmark/scenarios/sahara/test_jobs.py

@@ -17,7 +17,7 @@ import mock
 from oslo.config import cfg
 
 from rally.benchmark.scenarios.sahara import jobs
-from tests import test
+from tests.unit import test
 
 
 CONF = cfg.CONF

tests/unit/benchmark/scenarios/sahara/test_node_group_templates.py

@@ -16,7 +16,7 @@
 import mock
 
 from rally.benchmark.scenarios.sahara import node_group_templates as ngts
-from tests import test
+from tests.unit import test
 
 SAHARA_NGTS = ("rally.benchmark.scenarios.sahara.node_group_templates"
                ".SaharaNodeGroupTemplates")