diff --git a/bowling_ball/README.rst b/bowling_ball/README.rst
index c15c23db..329b6a25 100644
--- a/bowling_ball/README.rst
+++ b/bowling_ball/README.rst
@@ -11,24 +11,28 @@ This project aims to test for issues with rolling downtime on
OpenStack-Ansible deployments. It's comprised of two main components:
* The ``rolling_restart.py`` script
-* The ``tests`` directory
+* The ``rolling_test.py`` script
The ``rolling_restart.py`` script will stop containers from a specified group
in a rolling fashion - node 1 will stop, then start, then node 2, then
node 3 and so on. This script runs from the *deployment host*.
The ``tests`` directory contains scripts to generate traffic against the
-target services. These vary per service, but attempt to apply usage to a
+target services.
system that will be restarted by ``rolling_restart.py`` in order to
measure the effects. These scripts run from a *utility container*.
+The ``rolling_test.py`` script contains tests to generate traffic against the
+target services. These vary per service, but attempt to apply usage to a
+system that will be restarted by ``rolling_restart.py`` in order to
+measure the effects. This script runs from a *utility container*.
Usage
-----
-#. Start your test script from the utility container. ``keystone.py``
- will request a session and a list of projects on an infinite loop, for
- example.
+#. Start your test from a utility container. ``./rolling_test.py keystone``
+ runs the Keystone test. ``./rolling_test.py list`` will list tests and
+ their descriptions
#. From the deployment node, run ``rolling_restart.py`` in the playbooks
directory (necessary to find the inventory script). Specify the service
you're targeting with the ``-s`` parameter.
@@ -48,12 +52,34 @@ These tools are currently coupled to OSA, and they assume paths to files
as specified by the ``multi-node-aio`` scripts.
Container stopping and starting is done with an ansible command, and the
-physical host to target is derivced from the current inventory.
+physical host to target is derived from the current inventory.
``rolling_restart.py`` must currently be run from the ``playbooks``
directory. This will be fixed later.
-You must source ``openrc`` before running ``keystone.py``.
+You must source ``openrc`` before running ``rolling_test.py``.
+
+
+Creating New Tests
+------------------
+
+Tests should subclass from the ``ServiceTest`` class in the same file
+and implement the following properties and methods:
+
+#. ``run`` - The actual test to run should be placed in this method. Timings
+ will be gathered based on when this function starts and stops.
+
+#. ``pre_test`` - Any pre-test setup that needs to happen, like creating a
+ file for Glance, Cinder, or Swift upload.
+
+#. ``post_test`` - Any post-test teardown that might be needed.
+
+#. ``service_name`` - The primary service that is being tested.
+
+#. ``description`` - Brief description of what the test does.
+
+Finally, add the test to the ``available_tests`` dictionary with the
+invocation name as the key and the class as the value.
Why the name?
diff --git a/bowling_ball/rolling_tests.py b/bowling_ball/rolling_tests.py
old mode 100644
new mode 100755
index 5193c7a8..6d3eb031
--- a/bowling_ball/rolling_tests.py
+++ b/bowling_ball/rolling_tests.py
@@ -15,6 +15,7 @@
#
# (c) 2017, Nolan Brubaker <nolan.brubaker@rackspace.com>
+import argparse
import datetime
from keystoneauth1.identity import v3
from keystoneauth1 import session
@@ -57,31 +58,148 @@ def configure_logging(service):
logger.addHandler(logfile)
-def keystone_test(logger):
- configure_logging('keystone')
+class ServiceTest(object):
+ def pre_test(self, *args, **kwargs):
+ """Any actions that need to be taken before starting the timer
- auth_url = os.environ['OS_AUTH_URL']
- password = os.environ['OS_PASSWORD']
+ These actions will run inside the test loop, but before marking a
+ start time.
- auth = v3.Password(auth_url=auth_url, username="admin",
- password=password, project_name="admin",
- user_domain_id="default", project_domain_id="default")
- sess = session.Session(auth=auth)
- keystone = key_client.Client(session=sess)
- test_list = keystone.projects.list()
- if test_list:
- msg = "New project list."
- else:
- msg = "Failed to get project list"
- return msg
+ This might include creating a local resource, such as a file to upload
+ to Glance, Cinder, or Swift.
+
+ """
+ raise NotImplementedError
+
+ def run(self):
+ """Run the main test, within the timing window.
+
+ This test run should actually create and query a resource.
+ """
+ raise NotImplementedError
+
+ def post_test(self):
+ """Any post-test clean up work that needs to be done and not timed."""
+ raise NotImplementedError
+
+ def configure_logger(self, logger):
+ """Configure a stream and file log for a given service
+
+ :param: service - name of service for log file.
+ generates `/var/log/{service_name}_query.log`
+ :param: logger - logger to be configure for the test.
+ Filename will be based on the test's `service_name`
+ property
+ """
+ logger.setLevel(logging.INFO)
+ console = logging.StreamHandler()
+ filename = '/var/log/{}_rolling.log'.format(self.service_name)
+ logfile = logging.FileHandler(filename, 'a')
+
+ console.setLevel(logging.INFO)
+ logfile.setLevel(logging.INFO)
+
+ formatter = logging.Formatter(
+ '%(asctime)s - %(levelname)s - %(message)s')
+ # Make sure we're using UTC for everything.
+ formatter.converter = time.gmtime
+
+ console.setFormatter(formatter)
+ logfile.setFormatter(formatter)
+
+ logger.addHandler(console)
+ logger.addHandler(logfile)
+
+ # This is useful to a lot of tests, so implement it here for re-use
+ def get_session(self):
+ auth_url = os.environ['OS_AUTH_URL']
+ password = os.environ['OS_PASSWORD']
+ auth = v3.Password(auth_url=auth_url, username="admin",
+ password=password, project_name="admin",
+ user_domain_id="default",
+ project_domain_id="default")
+ sess = session.Session(auth=auth)
+ return sess
+
+ def get_keystone_client(self, session):
+ return key_client.Client(session=session)
-def test_loop(test_function):
+class KeystoneTest(ServiceTest):
+ service_name = 'keystone'
+ description = 'Obtain a token then a project list to validate it worked'
+
+ def run(self):
+
+ auth_url = os.environ['OS_AUTH_URL']
+ password = os.environ['OS_PASSWORD']
+
+ auth = v3.Password(auth_url=auth_url, username="admin",
+ password=password, project_name="admin",
+ user_domain_id="default",
+ project_domain_id="default")
+
+ sess = session.Session(auth=auth)
+ keystone = key_client.Client(session=sess)
+ test_list = keystone.projects.list()
+ if test_list:
+ msg = "New project list."
+ else:
+ msg = "Failed to get project list"
+ return msg
+
+
+class GlanceTest(ServiceTest):
+ service_name = 'glance'
+ description = 'Upload and delete a 1MB file'
+
+ def pre_test(self):
+ # make a bogus file to give to glance.
+ self.temp_file = tempfile.TemporaryFile()
+ self.temp_file.write(os.urandom(1024 * 1024))
+ self.temp_file.seek(0)
+
+ def run(self):
+ sess = self.get_session()
+ keystone = self.get_keystone_client(sess)
+ endpoint = self.get_glance_endpoint(keystone)
+
+ glance = Client(version='2', endpoint=endpoint, session=sess)
+ image = glance.images.create(name="Rolling test",
+ disk_format="raw",
+ container_format="bare")
+ try:
+ glance.images.upload(image.id, self.temp_file)
+ except exc.HTTPInternalServerError:
+ # TODO: set msg and error type instead.
+ logger.error("Failed to upload")
+ return
+ finally:
+ glance.images.delete(image.id)
+ self.temp_file.close()
+
+ msg = "Image created and deleted."
+ return msg
+
+ def get_glance_endpoint(self, keystone):
+ """Get the glance admin endpoint
+
+ Because we don't want to set up SSL handling, use the plain HTTP
+ endpoints.
+ """
+ service_id = keystone.services.find(name='glance')
+ glance_endpoint = keystone.endpoints.list(service=service_id,
+ interface='admin')[0]
+ # The glance client wants the URL, not the keystone object
+ return glance_endpoint.url
+
+
+def test_loop(test):
"""Main loop to execute tests
Executes and times interactions with OpenStack services to gather timing
data.
- :param: test_function - function object that performs some action
+ :param: test - on object that performs some action
against an OpenStack service API.
"""
disconnected = None
@@ -97,10 +215,16 @@ def test_loop(test_function):
# Pause for a bit so we're not generating more data than we
# can handle
time.sleep(1)
+
+ try:
+ test.pre_test()
+ except NotImplementedError:
+ pass
+
start_time = datetime.datetime.now()
# Let the test function report it's own errors
- msg = test_function(logger)
+ msg = test.run()
end_time = datetime.datetime.now()
@@ -112,7 +236,13 @@ def test_loop(test_function):
delta = end_time - start_time
- logger.info("{}s {}s.".format(msg, delta.total_seconds()))
+ logger.info("{} {}".format(msg, delta.total_seconds()))
+
+ try:
+ test.post_test()
+ except NotImplementedError:
+ pass
+
except (exc_list):
if not disconnected:
disconnected = datetime.datetime.now()
@@ -120,62 +250,45 @@ def test_loop(test_function):
sys.exit()
-def get_session():
- auth_url = os.environ['OS_AUTH_URL']
- password = os.environ['OS_PASSWORD']
- auth = v3.Password(auth_url=auth_url, username="admin",
- password=password, project_name="admin",
- user_domain_id="default",
- project_domain_id="default")
- sess = session.Session(auth=auth)
- return sess
+available_tests = {
+ 'keystone': KeystoneTest,
+ 'glance': GlanceTest,
+}
-def get_keystone_client(session):
- return key_client.Client(session=session)
+def args(arg_list):
+
+ parser = argparse.ArgumentParser(
+ usage='%(prog)s',
+ description='OpenStack activity simulators',
+ )
+
+ parser.add_argument(
+ 'test',
+ help=("Name of test to execute, 'list' for a list of available"
+ " tests")
+ )
+ return parser.parse_args(arg_list)
-def get_glance_endpoint(keystone):
- """Get the glance admin endpoint
+def find_test(test_name):
+ if test_name in available_tests:
+ return available_tests[test_name]
+ elif test_name == "list":
+ for key, test_class in available_tests.items():
+ print("{} -> {}".format(key, test_class.description))
+ sys.exit()
+ else:
+ print("Test named {} not found.".format(test_name))
+ sys.exit()
- Because we don't want to set up SSL handling, use the plain HTTP
- endpoints.
- """
- service_id = keystone.services.find(name='glance')
- glance_endpoint = keystone.endpoints.list(service=service_id,
- interface='admin')[0]
- # The glance client wants the URL, not the keystone object
- return glance_endpoint.url
-
-
-def glance_test(logger):
- configure_logging('glance')
- # make a bogus file to give to glance.
-
- sess = get_session()
- keystone = get_keystone_client(sess)
- endpoint = get_glance_endpoint(keystone)
-
- temp_file = tempfile.TemporaryFile()
- temp_file.write(os.urandom(1024 * 1024))
- temp_file.seek(0)
-
- glance = Client(version='2', endpoint=endpoint, session=sess)
- image = glance.images.create(name="Rolling test",
- disk_format="raw",
- container_format="bare")
- try:
- glance.images.upload(image.id, temp_file)
- except exc.HTTPInternalServerError:
- # TODO: set msg and error type instead.
- logger.error("Failed to upload")
- return
- finally:
- glance.images.delete(image.id)
- temp_file.close()
-
- msg = "Image created and deleted."
- return msg
if __name__ == "__main__":
- test_loop(glance_test)
+ all_args = args(sys.argv[1:])
+
+ target_test_class = find_test(all_args.test)
+
+ target_test = target_test_class()
+ target_test.configure_logger(logger)
+
+ test_loop(target_test)