Add oslo libraries and command-line utility

Add oslo libraries for works with database and config.
Refactor database relation code. Move it in refstack/db.

Now after istallation of refstack available a command-line
utility: `refstack-manage` and `refstack-api`.

`refstack-manage`

This utlity provide commands for manipulation of database
migrations and setup config file through
--config-file /path/to/refstack.conf option.

Config file should contain database connection string.
For example:

[DEFAULT]
sql_connection = mysql://root:passw0rd@127.0.0.1/refstack

Add ability to runtime updrade/downgrade db through command-line utillity.

`refstack-api`

This utility allow to launch api on gunicorn server. Specify config through
--env REFSTACK_OSLO_CONFIG=/path/to/refstack.conf

Change-Id: Ibea97d433d54d307233aa04e9487f1a1230e7487

bin/refstack-api

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Command-line launcher for Refstack API
+"""
+
+import sys
+
+from pecan.commands import serve
+
+from refstack.api import config as api_config
+
+
+def get_pecan_config():
+    """ Get path to pecan configuration file """
+    filename = api_config.__file__.replace('.pyc', '.py')
+    return filename
+
+
+if __name__ == '__main__':
+    config_path = get_pecan_config()
+    sys.argv.append(config_path)
+    serve.gunicorn_run()

bin/refstack-manage

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Command-line utility for database manage
+"""
+
+import sys
+
+from oslo.config import cfg
+
+from refstack.db import migration
+
+CONF = cfg.CONF
+
+
+class DatabaseManager(object):
+
+    def version(self):
+        print(migration.version())
+
+    def upgrade(self):
+        migration.upgrade(CONF.command.revision)
+
+    def downgrade(self):
+        migration.downgrade(CONF.command.revision)
+
+    def stamp(self):
+        migration.stamp(CONF.command.revision)
+
+    def revision(self):
+        migration.revision(CONF.command.message, CONF.command.autogenerate)
+
+
+def add_command_parsers(subparsers):
+    db_manager = DatabaseManager()
+
+    parser = subparsers.add_parser('version',
+                                   help='show current database version')
+    parser.set_defaults(func=db_manager.version)
+
+    parser = subparsers.add_parser('upgrade',
+                                   help='upgrade database to '
+                                        'the specified version')
+    parser.set_defaults(func=db_manager.upgrade)
+    parser.add_argument('--revision', nargs='?',
+                        help='desired database version')
+
+    parser = subparsers.add_parser('downgrade',
+                                   help='downgrade database '
+                                        'to the specified version')
+    parser.set_defaults(func=db_manager.downgrade)
+    parser.add_argument('--revision', nargs='?',
+                        help='desired database version')
+
+    parser = subparsers.add_parser('stamp',
+                                   help='stamp database with provided '
+                                        'revision. Don\'t run any migrations')
+    parser.add_argument('--revision', nargs='?',
+                        help='should match one from repository or head - '
+                             'to stamp database with most recent revision')
+    parser.set_defaults(func=db_manager.stamp)
+
+    parser = subparsers.add_parser('revision',
+                                   help='create template for migration')
+    parser.add_argument('-m', '--message',
+                        help='text that will be used for migration title')
+    parser.add_argument('--autogenerate', action='store_true',
+                        help='if True - generates diff based '
+                             'on current database state (True by default)')
+    parser.set_defaults(func=db_manager.revision)
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Available commands',
+                                handler=add_command_parsers)
+
+CONF.register_cli_opt(command_opt)
+
+if __name__ == '__main__':
+    CONF(sys.argv[1:], project='refstack')
+    CONF.command.func()

doc/refstack.md

@@ -1,8 +1,5 @@
 Refstack Quickstart
 ===================
-
-Instruction to run refstack for development or behind your firewall.
-
 ####Install dependencies (on ubuntu 14.x)..
 
 - `sudo apt-get install git python-dev libssl-dev python-setuptools`
@@ -46,40 +43,44 @@ Instruction to run refstack for development or behind your firewall.
 
 - `cd refstack`
 
-- Update the db connection strings in following files to the correct
- information of your environment.
-
- - The `sqlalchemy.url = mysql://root:passw0rd@127.0.0.1/refstack` string
-   in the `./refstack/db/migrations/alembic.ini` file.
-
- - The `'db_url': 'mysql://root:passw0rd@127.0.0.1/refstack'` string in the
-   `./refstack/api/config.py` file.
-
- - NOTE: You may need to also update the `'debug': False` string in the
-   `./refstack/api/config.py` file for development.
-
 - Creare virtual environment: `virtualenv .venv --system-site-package`
 
 - Source to virtual environment: `source .venv/bin/activate`
 
-- Install refstack: `python setup.py install`
 
-- Create tables in the refstack database.
+####Install Refstack application (on ubuntu 14.x)..
 
- - `cd ./refstack/db/migrations/`
+- `python setup.py install`
 
- - `alembic upgrade head`
+####Configuration file preparation
 
- - `cd ../../..`
+- Make a copy of the sample config and update it with the correct information of your environment. Example of config file available in etc directory.
 
-Plug this bad boy into your server infrastructure.
+####Database sync
 
-We use nginx and gunicorn, you may use something else if you so desire.
+- Check current revision:
+
+  `refstack-manage --config-file /path/to/refstack.conf version`
+
+  The response will show the current database revision. If the revision is `None` (indicating a clear database), the following command should be performed to upgrade the database to the latest revision:
+
+ - Upgrade database to latest revision:
+
+   `refstack-manage --config-file /path/to/refstack.conf upgrade --revision head`
+
+ - Check current revision:
+
+   `refstack-manage --config-file /path/to/refstack.conf version`
+
+   Now it should be `42278d6179b9`.
+
+
+####Start Refstack
 
 For the most basic setup that you can try right now, just kick off
 gunicorn:
 
-`gunicorn_pecan --debug refstack/api/config.py`
+- `refstack-api --env REFSTACK_OSLO_CONFIG=/path/to/refstack.conf`
 
 Now available http://localhost:8000/ with JSON response {'Root': 'OK'}
-and http://localhost:8000/v1/results/ with JSON response {'Results': 'OK'}.
+and http://localhost:8000/v1/results/ with JSON response {'Results': 'OK'}.

etc/refstack.conf.sample

@@ -0,0 +1,2 @@
+[DEFAULT]
+sql_connection = mysql://refstack:<your password>@127.0.0.1/refstack

refstack/api/app.py

@@ -17,31 +17,18 @@
 
 import json
 import logging
+import os
 
+from oslo.config import cfg
 import pecan
 from pecan import hooks
 import webob
 
-from refstack import backend
+from refstack import utils
 
 logger = logging.getLogger(__name__)
 
-
-class BackendHook(hooks.PecanHook):
-
-    """Pecan Hook for providing backend functionality."""
-
-    def __init__(self, app_config):
-        """Hook init."""
-        self.global_backend = backend.Backend(app_config)
-
-    def before(self, state):
-        """Before request."""
-        state.request.backend = self.global_backend.create_local()
-
-    def after(self, state):
-        """After request."""
-        pass
+CONF = cfg.CONF
 
 
 class JSONErrorHook(hooks.PecanHook):
@@ -85,9 +72,28 @@ def setup_app(config):
         app_conf.pop('root'),
         logging=getattr(config, 'logging', {}),
         hooks=[JSONErrorHook(app_conf), hooks.RequestViewerHook(
-            {'items': ['status', 'method', 'controller', 'path']}
-        ), BackendHook(app_conf)],
+            {'items': ['status', 'method', 'controller', 'path', 'body']},
+            headers=False, writer=utils.LogWriter(logger, logging.DEBUG)
+        )],
         **app_conf
     )
 
+    # By default we expect path to oslo config file in environment variable
+    # REFSTACK_OSLO_CONFIG (option for testing and development)
+    # If it is empty we look up those config files
+    # in the following directories:
+    #   ~/.${project}/
+    #   ~/
+    #   /etc/${project}/
+    #   /etc/
+    default_config_files = ((os.getenv('REFSTACK_OSLO_CONFIG'), )
+                            if os.getenv('REFSTACK_OSLO_CONFIG')
+                            else cfg.find_config_files('refstack'))
+
+    CONF('',
+         project='refstack',
+         default_config_files=default_config_files)
+
+    CONF.log_opt_values(logger, logging.DEBUG)
+
     return app

refstack/api/config.py

@@ -34,7 +34,6 @@ server = {
 app = {
     'root': 'refstack.api.controllers.root.RootController',
     'modules': ['refstack.api'],
-    'db_url': 'mysql://root:r00t@127.0.0.1/refstack',
     'static_root': '%(confdir)s/../static',
     'template_path': '%(confdir)s/../templates',
     # The 'debug' option should be false in production servers, but needs to be

refstack/api/controllers/v1.py

@@ -19,6 +19,8 @@ import logging
 import pecan
 from pecan import rest
 
+from refstack import db
+
 logger = logging.getLogger(__name__)
 
 
@@ -37,11 +39,11 @@ class ResultsController(rest.RestController):
 
         :param test_id: ID of the test to get the JSON for.
         """
-        test_info = pecan.request.backend.get_test(test_id)
+        test_info = db.get_test(test_id)
         if not test_info:
             pecan.abort(404)
 
-        test_list = pecan.request.backend.get_test_results(test_id)
+        test_list = db.get_test_results(test_id)
         test_name_list = [test_dict[0] for test_dict in test_list]
         return {"cpid": test_info.cpid,
                 "created_at": test_info.created_at,
@@ -58,7 +60,7 @@ class ResultsController(rest.RestController):
                                detail='Request body \'%s\' could not '
                                       'be decoded as JSON.'
                                       '' % pecan.request.body)
-        test_id = pecan.request.backend.store_results(results)
+        test_id = db.store_results(results)
         return {'test_id': test_id}
 
 

refstack/backend.py

@@ -1,88 +0,0 @@
-#
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""Backend provider."""
-
-import logging
-import uuid
-
-import sqlalchemy as sa
-from sqlalchemy import orm
-
-from refstack import models
-
-logger = logging.getLogger(__name__)
-
-
-class Backend(object):
-
-    """Global backend provider."""
-
-    def __init__(self, app_config):
-        """Backend factory."""
-        engine = sa.create_engine(app_config['db_url'])
-        self.session_maker = orm.sessionmaker()
-        self.session_maker.configure(bind=engine)
-
-    def create_local(self):
-        """Create request-local Backend instance."""
-        return LocalBackend(self)
-
-
-class LocalBackend(object):
-
-    """Request-local backend provider."""
-
-    def __init__(self, global_backend):
-        """Request-local backend instance."""
-        self.db_session = global_backend.session_maker()
-
-    def store_results(self, results):
-        """Storing results into database.
-
-        :param results: Dict describes test results.
-        """
-        session = self.db_session
-
-        test_id = str(uuid.uuid4())
-        test = models.Test(id=test_id, cpid=results.get('cpid'),
-                           duration_seconds=results.get('duration_seconds'))
-        test_results = results.get('results', [])
-        for result in test_results:
-            session.add(models.TestResults(
-                test_id=test_id, name=result['name'],
-                uid=result.get('uid', None)
-            ))
-        session.add(test)
-        session.commit()
-        return test_id
-
-    def get_test(self, test_id):
-        """Get test information from the database.
-
-        :param test_id: The ID of the test.
-        """
-        test_info = self.db_session.query(models.Test).\
-            filter_by(id=test_id).first()
-        return test_info
-
-    def get_test_results(self, test_id):
-        """Get all passed tempest tests for a particular test.
-
-        :param test_id: The ID of the test.
-        """
-        results = self.db_session.query(models.TestResults.name).filter_by(
-            test_id=test_id).all()
-        return results

refstack/db/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+DB abstraction for Refstack
+"""
+
+from refstack.db.api import *  # noqa

refstack/db/api.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Defines interface for DB access.
+
+Functions in this module are imported into the refstack.db namespace.
+Call these functions from refstack.db namespace, not the refstack.db.api
+namespace.
+
+"""
+from oslo.config import cfg
+from oslo.db import api as db_api
+
+
+db_opts = [
+    cfg.StrOpt('db_backend',
+               default='sqlalchemy',
+               help='The backend to use for database.'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(db_opts)
+
+_BACKEND_MAPPING = {'sqlalchemy': 'refstack.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING,
+                                lazy=True)
+
+
+###################
+def store_results(results):
+    """Storing results into database.
+
+    :param results: Dict describes test results.
+    """
+    return IMPL.store_results(results)
+
+
+def get_test(test_id):
+    """Get test information from the database.
+
+    :param test_id: The ID of the test.
+    """
+    return IMPL.get_test(test_id)
+
+
+def get_test_results(test_id):
+    """Get all passed tempest tests for a particular test.
+
+    :param test_id: The ID of the test.
+    """
+    return IMPL.get_test_results(test_id)

refstack/db/migration.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from refstack.db import utils as db_utils
+
+
+IMPL = db_utils.PluggableBackend(
+    'db_backend', sqlalchemy='refstack.db.migrations.alembic.migration')
+
+
+def version():
+    """Display the current database version."""
+    return IMPL.version()
+
+
+def upgrade(version):
+    """Upgrade database to 'version' or the most recent version."""
+    return IMPL.upgrade(version)
+
+
+def downgrade(version):
+    """Downgrade database to 'version' or to initial state."""
+    return IMPL.downgrade(version)
+
+
+def stamp(version):
+    """Stamp database with 'version' or the most recent version."""
+    return IMPL.stamp(version)
+
+
+def revision(message, autogenerate):
+    """Generate new migration script."""
+    return IMPL.revision(message, autogenerate)

refstack/db/migrations/alembic.ini

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = alembic
+script_location = %(here)s/alembic
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
@@ -12,7 +12,6 @@ script_location = alembic
 # revision_environment = false
 
 #sqlalchemy.url = driver://user:pass@127.0.0.1/dbname
-sqlalchemy.url = mysql://root:r00t@127.0.0.1/refstack
 
 # Logging configuration
 [loggers]

refstack/db/migrations/alembic/env.py

@@ -17,54 +17,21 @@
 from __future__ import with_statement
 
 from alembic import context
-from sqlalchemy import engine_from_config, pool
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-from refstack.models import Base
-target_metadata = Base.metadata
-# target_metadata = None
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline():
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
-
-    with context.begin_transaction():
-        context.run_migrations()
+from refstack.db.sqlalchemy import api as db_api
+from refstack.db.sqlalchemy import models as db_models
 
 
 def run_migrations_online():
+
     """Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine
     and associate a connection with the context."""
-    engine = engine_from_config(
-        config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool)
 
+    engine = db_api.get_engine()
     connection = engine.connect()
+    target_metadata = db_models.RefStackBase.metadata
     context.configure(
         connection=connection,
         target_metadata=target_metadata)
@@ -76,7 +43,4 @@ def run_migrations_online():
         connection.close()
 
 
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()
+run_migrations_online()

refstack/db/migrations/alembic/migration.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Implementation of Alembic commands.
+"""
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo.config import cfg
+
+from refstack.db.sqlalchemy import api as db_api
+
+CONF = cfg.CONF
+
+
+def _alembic_config():
+    path = os.path.join(os.path.dirname(__file__), os.pardir, 'alembic.ini')
+    config = alembic_config.Config(path)
+    return config
+
+
+def version():
+    """Current database version.
+
+    :returns: Database version
+    :type: string
+    """
+    engine = db_api.get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
+
+
+def upgrade(revision):
+    """Upgrade database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    return alembic.command.upgrade(_alembic_config(), revision or 'head')
+
+
+def downgrade(revision):
+    """Downgrade database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    return alembic.command.downgrade(_alembic_config(), revision or 'base')
+
+
+def stamp(revision):
+    """Stamp database with provided revision.
+
+    Don't run any migrations.
+
+    :param revision: Should match one from repository or head - to stamp
+    database with most recent revision
+    :type revision: string
+    """
+    return alembic.command.stamp(_alembic_config(), revision or 'head')
+
+
+def revision(message=None, autogenerate=False):
+    """Create template for migration.
+
+    :param message: Text that will be used for migration title
+    :type message: string
+    :param autogenerate: If True - generates diff based on current database
+    state
+    :type autogenerate: bool
+    """
+    return alembic.command.revision(_alembic_config(), message, autogenerate)

refstack/db/migrations/alembic/versions/42278d6179b9_init.py

@@ -18,6 +18,9 @@ def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         'test',
+        sa.Column('updated_at', sa.DateTime()),
+        sa.Column('deleted_at', sa.DateTime()),
+        sa.Column('deleted', sa.Integer, default=0),
         sa.Column('id', sa.String(length=36), nullable=False),
         sa.Column('created_at', sa.DateTime(), nullable=False),
         sa.Column('cpid', sa.String(length=128), nullable=False),
@@ -26,7 +29,11 @@ def upgrade():
     )
     op.create_table(
         'meta',
+        sa.Column('updated_at', sa.DateTime()),
+        sa.Column('deleted_at', sa.DateTime()),
+        sa.Column('deleted', sa.Integer, default=0),
         sa.Column('_id', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False),
         sa.Column('test_id', sa.String(length=36), nullable=False),
         sa.Column('meta_key', sa.String(length=64), nullable=False),
         sa.Column('value', sa.Text(), nullable=True),
@@ -36,7 +43,11 @@ def upgrade():
     )
     op.create_table(
         'results',
+        sa.Column('updated_at', sa.DateTime()),
+        sa.Column('deleted_at', sa.DateTime()),
+        sa.Column('deleted', sa.Integer, default=0),
         sa.Column('_id', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False),
         sa.Column('test_id', sa.String(length=36), nullable=False),
         sa.Column('name', sa.String(length=512), nullable=True),
         sa.Column('uid', sa.String(length=36), nullable=True),

refstack/db/sqlalchemy/api.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Implementation of SQLAlchemy backend.
+"""
+import sys
+import uuid
+
+from oslo.config import cfg
+from oslo.db import options as db_options
+from oslo.db.sqlalchemy import session as db_session
+
+from refstack.db.sqlalchemy import models
+
+
+CONF = cfg.CONF
+
+_FACADE = None
+
+_DEFAULT_SQL_CONNECTION = 'sqlite://'
+db_options.set_defaults(cfg.CONF,
+                        connection=_DEFAULT_SQL_CONNECTION)
+
+
+def _create_facade_lazily():
+    global _FACADE
+    if _FACADE is None:
+        _FACADE = db_session.EngineFacade.from_config(CONF)
+    return _FACADE
+
+
+def get_engine():
+    facade = _create_facade_lazily()
+    return facade.get_engine()
+
+
+def get_session(**kwargs):
+    facade = _create_facade_lazily()
+    return facade.get_session(**kwargs)
+
+
+def get_backend():
+    """The backend is this module itself."""
+
+    return sys.modules[__name__]
+
+
+###################
+
+
+def store_results(results):
+    test = models.Test()
+    test_id = str(uuid.uuid4())
+    test.id = test_id
+    test.cpid = results.get('cpid')
+    test.duration_seconds = results.get('duration_seconds')
+
+    received_test_results = results.get('results', [])
+    session = get_session()
+    with session.begin():
+        test.save(session)
+        for result in received_test_results:
+            test_result = models.TestResults()
+            test_result.test_id = test_id
+            test_result.name = result['name']
+            test_result.uid = result.get('uid', None)
+            test_result.save(session)
+    return test_id
+
+
+def get_test(test_id):
+    session = get_session()
+    test_info = session.query(models.Test).\
+        filter_by(id=test_id).\
+        first()
+    return test_info
+
+
+def get_test_results(test_id):
+    session = get_session()
+    results = session.query(models.TestResults.name).\
+        filter_by(test_id=test_id).\
+        all()
+    return results

refstack/db/sqlalchemy/models.py

@@ -13,34 +13,56 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
-"""DB models"""
+"""
+SQLAlchemy models for Refstack data.
+"""
 
 import datetime
 
+from oslo.config import cfg
+from oslo.db.sqlalchemy import models
+from oslo.utils import timeutils
 import sqlalchemy as sa
 from sqlalchemy import orm
 from sqlalchemy.ext.declarative import declarative_base
 
-Base = declarative_base()
+CONF = cfg.CONF
+BASE = declarative_base()
 
 
-class Test(Base):
+class RefStackBase(models.ModelBase, models.TimestampMixin):
+
+    """Base class for RefStack Models."""
+
+    __table_args__ = {'mysql_engine': 'InnoDB'}
+    created_at = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow,
+                           nullable=False)
+    updated_at = sa.Column(sa.DateTime())
+    deleted_at = sa.Column(sa.DateTime)
+    deleted = sa.Column(sa.Integer, default=0)
+    metadata = None
+
+    def delete(self, session=None):
+        """Delete this object."""
+        self.deleted = self.id
+        self.deleted_at = timeutils.utcnow()
+        self.save(session=session)
+
+
+class Test(BASE, RefStackBase):
 
     """Test."""
 
     __tablename__ = 'test'
 
     id = sa.Column(sa.String(36), primary_key=True)
-    created_at = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow,
-                           nullable=False)
     cpid = sa.Column(sa.String(128), index=True, nullable=False)
     duration_seconds = sa.Column(sa.Integer, nullable=False)
     results = orm.relationship('TestResults', backref='test')
     meta = orm.relationship('TestMeta', backref='test')
 
 
-class TestResults(Base):
+class TestResults(BASE, RefStackBase):
 
     """Test results."""
 
@@ -56,7 +78,7 @@ class TestResults(Base):
     uid = sa.Column(sa.String(36))
 
 
-class TestMeta(Base):
+class TestMeta(BASE, RefStackBase):
 
     """Test metadata."""
 

refstack/db/utils.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2015 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utilities for database."""
+
+import logging
+
+from oslo.config import cfg
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class PluggableBackend(object):
+    """A pluggable backend loaded lazily based on some value."""
+
+    def __init__(self, pivot, **backends):
+        self.__backends = backends
+        self.__pivot = pivot
+        self.__backend = None
+
+    def __get_backend(self):
+        if not self.__backend:
+            backend_name = CONF[self.__pivot]
+            if backend_name not in self.__backends:
+                raise Exception('Invalid backend: %s' % backend_name)
+
+            backend = self.__backends[backend_name]
+            if isinstance(backend, tuple):
+                name = backend[0]
+                fromlist = backend[1]
+            else:
+                name = backend
+                fromlist = backend
+
+            self.__backend = __import__(name, None, None, fromlist)
+            LOG.debug('backend %s', self.__backend)
+        return self.__backend
+
+    def __getattr__(self, key):
+        backend = self.__get_backend()
+        return getattr(backend, key)

refstack/tests/api/__init__.py

@@ -19,14 +19,16 @@ import os
 
 import alembic
 import alembic.config
-from pecan import set_config
-from pecan.testing import load_test_app
+from oslo.config import cfg
 import sqlalchemy as sa
 import sqlalchemy.exc
 from unittest import TestCase
+from webtest import TestApp
 
 import refstack
-from refstack.models import Base
+from refstack.api import app
+
+CONF = cfg.CONF
 
 
 class FunctionalTest(TestCase):
@@ -39,33 +41,32 @@ class FunctionalTest(TestCase):
 
     def setUp(self):
         """Test setup."""
-        self.config = {
-            'app': {
+        class TestConfig(object):
+            app = {
                 'root': 'refstack.api.controllers.root.RootController',
-                'db_url': os.environ.get(
-                    'TEST_DB_URL',
-                    'mysql://root:r00t@127.0.0.1/refstack_test'
-                ),
                 'modules': ['refstack.api'],
                 'static_root': '%(confdir)s/public',
                 'template_path': '%(confdir)s/${package}/templates',
             }
-        }
+
+        test_config = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            'refstack.test.conf'
+        )
+        os.environ['REFSTACK_OSLO_CONFIG'] = test_config
         self.project_path = os.path.abspath(
             os.path.join(inspect.getabsfile(refstack), '..', '..'))
-
+        self.app = TestApp(app.setup_app(TestConfig()))
         self.prepare_test_db()
         self.migrate_test_db()
 
-        self.app = load_test_app(self.config)
-
     def tearDown(self):
         """Test teardown."""
-        set_config({}, overwrite=True)
+        self.app.reset()
 
     def prepare_test_db(self):
         """Create/clear test database."""
-        db_url = self.config['app']['db_url']
+        db_url = CONF.database.connection
         db_name = db_url.split('/')[-1]
         short_db_url = '/'.join(db_url.split('/')[0:-1])
         try:
@@ -83,21 +84,16 @@ class FunctionalTest(TestCase):
             conn.execute('create database %s' % db_name)
             conn.close()
 
-        engine = sa.create_engine(db_url)
-        conn = engine.connect()
-        conn.execute('commit')
-        for tbl in reversed(Base.metadata.sorted_tables):
-            if engine.has_table(tbl.name):
-                conn.execute('drop table %s' % tbl.name)
-        conn.close()
-
     def migrate_test_db(self):
         """Apply migrations to test database."""
         alembic_cfg = alembic.config.Config()
-        alembic_cfg.set_main_option("script_location",
-                                    os.path.join(self.project_path, 'alembic'))
+        alembic_cfg.set_main_option(
+            "script_location",
+            os.path.join(self.project_path, 'refstack', 'db',
+                         'migrations', 'alembic')
+        )
         alembic_cfg.set_main_option("sqlalchemy.url",
-                                    self.config['app']['db_url'])
+                                    CONF.database.connection)
         alembic.command.upgrade(alembic_cfg, 'head')
 
     def get_json(self, url, headers=None, extra_environ=None,

refstack/tests/api/refstack.test.conf

@@ -0,0 +1,2 @@
+[DEFAULT]
+sql_connection = mysql://root:passw0rd@127.0.0.1/refstack

refstack/utils.py

@@ -20,6 +20,7 @@
 from datetime import datetime
 import os
 import random
+import re
 import string
 
 
@@ -60,6 +61,18 @@ SEX_TYPE = {
 STRING_LEN = 64
 
 
+class LogWriter(object):
+    """Stream-like API to logger"""
+
+    def __init__(self, logger, level):
+        self.logger = logger
+        self.level = level
+
+    def write(self, s):
+        if re.sub('[\n ]', '', s):
+            self.logger.log(self.level, '\n' + s)
+
+
 def get_current_time():
     return datetime.utcnow()
 

requirements.txt

@@ -1,6 +1,9 @@
 SQLAlchemy==0.8.3
 alembic==0.5.0
-gunicorn==0.17.4
+#gunicorn 19.1.1 has a bug with threading module
+gunicorn==18
+oslo.config>=1.6.0 # Apache-2.0
+oslo.db>=1.4.1 # Apache-2.0
 pecan>=0.8.2
 pyOpenSSL==0.13
 pycrypto==2.6

setup.cfg

@@ -22,7 +22,10 @@ classifier =
 packages =
     refstack
 
+scripts =
+    bin/refstack-manage
+    bin/refstack-api
+
 [global]
 setup-hooks =
     pbr.hooks.setup_hook
-

tox.ini

@@ -31,7 +31,9 @@ commands = python -m unittest discover ./refstack/tests/api
 distribute = false
 
 [testenv:pep8]
-commands = flake8
+commands =
+  flake8 {posargs}
+  flake8 --filename=refstack* bin
 distribute = false
 
 [testenv:venv]