Integrate OSprofiler and Glance

*) Add osprofiler wsgi middleware
This middleware is used for 2 things:
1) It checks that person who want to trace is trusted and knows
secret HMAC key.
2) It start tracing in case of proper trace headers
and add first wsgi trace point, with info about HTTP request

*) Add initialization of osprofiler at start of serivce
Set's olso.messaging notifer instance (to send notifications to
Ceilometer)

*) Add tracing support for sqlalchemy

*) Add profiler CONF group that has to options:
1) Enable or disable profiler
2) Enable or disable sqlalchemy tracing
(the reason why we put this in Conf options, is that usually
db requests create a lot of trace info)

*) Glance-registry wasn't setted properly to send notifications.

To test it you need 2 things:
1) Enable ceilometer in devstack:
http://paste.openstack.org/show/85724/
2) Initialize osprofiler and send out API request. You can use
one of follow approaches:
a. Run this script (pls provide proper credentials)
http://paste.openstack.org/show/85722/
b. Use the function https://review.openstack.org/#/c/111184/
added from glanceclient CLI directly.

DocImpact

Change-Id: I45a19f5eb5304c2b78a9e12cbc0744941a807304
Co-author: Zhi Yan Liu <zhiyanl@cn.ibm.com>#

doc/source/configuring.rst

@@ -1268,3 +1268,37 @@ the ``failure`` state.
 * ``task_time_to_live=<Time_in_hours>``
 
 Optional. Default: ``48``
+
+Configuring Glance performance profiling
+----------------------------------------
+
+Glance supports using osprofiler to trace the performance of each key internal
+handling, including RESTful API calling, DB operation and etc.
+
+``Please be aware that Glance performance profiling is currently a work in
+progress feature.`` Although, some trace points is available, e.g. API
+execution profiling at wsgi main entry and SQL execution profiling at DB
+module, the more fine-grained trace point is being worked on.
+
+The config value ``enabled`` is used to determine whether fully enable
+profiling feature for glance-api and glance-registry service.
+
+* ``enabled=<True|False>``
+
+Optional. Default: ``True``
+
+The config value ``trace_sqlalchemy`` is used to determin whether fully enable
+sqlalchemy engine based SQL execution profiling feature for glance-api and
+glance-registry services.
+
+* ``trace_sqlalchemy=<True|False>``
+
+Optional. Default: ``True``
+
+**IMPORTANT NOTE**: The HMAC key which is used for encrypting context data for
+performance profiling is configued in paste config file of glance-api and
+glance-registry service separately, by default they place at
+/etc/glance/api-paste.ini and /etc/glance/registry-paste.ini files, in order
+to make profiling work as designed operator needs to make those values of HMAC
+key be consistent for all services in your deployment. Without HMAC key the
+profiling will not be triggered even profiling feature is enabled.

etc/glance-api-paste.ini

@@ -1,38 +1,38 @@
 # Use this pipeline for no auth or image caching - DEFAULT
 [pipeline:glance-api]
-pipeline = versionnegotiation unauthenticated-context rootapp
+pipeline = versionnegotiation unauthenticated-context osprofiler rootapp
 
 # Use this pipeline for image caching and no auth
 [pipeline:glance-api-caching]
-pipeline = versionnegotiation unauthenticated-context cache rootapp
+pipeline = versionnegotiation unauthenticated-context osprofiler cache rootapp
 
 # Use this pipeline for caching w/ management interface but no auth
 [pipeline:glance-api-cachemanagement]
-pipeline = versionnegotiation unauthenticated-context cache cachemanage rootapp
+pipeline = versionnegotiation unauthenticated-context osprofiler cache cachemanage rootapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-api-keystone]
-pipeline = versionnegotiation authtoken context rootapp
+pipeline = versionnegotiation authtoken context osprofiler rootapp
 
 # Use this pipeline for keystone auth with image caching
 [pipeline:glance-api-keystone+caching]
-pipeline = versionnegotiation authtoken context cache rootapp
+pipeline = versionnegotiation authtoken context osprofiler cache rootapp
 
 # Use this pipeline for keystone auth with caching and cache management
 [pipeline:glance-api-keystone+cachemanagement]
-pipeline = versionnegotiation authtoken context cache cachemanage rootapp
+pipeline = versionnegotiation authtoken context osprofiler cache cachemanage rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-api-trusted-auth]
-pipeline = versionnegotiation context rootapp
+pipeline = versionnegotiation context osprofiler rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user and uses cache management
 [pipeline:glance-api-trusted-auth+cachemanagement]
-pipeline = versionnegotiation context cache cachemanage rootapp
+pipeline = versionnegotiation context osprofiler cache cachemanage rootapp
 
 [composite:rootapp]
 paste.composite_factory = glance.api:root_app_factory
@@ -70,3 +70,8 @@ delay_auth_decision = true
 
 [filter:gzip]
 paste.filter_factory = glance.api.middleware.gzip:GzipMiddleware.factory
+
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
+hmac_keys = SECRET_KEY
+enabled = yes

etc/glance-api.conf

@@ -687,3 +687,10 @@ admin_password = %SERVICE_PASSWORD%
 # This option will be applied when you using 'store_type' option as image
 # location strategy defined by the 'location_strategy' config option.
 #store_type_preference =
+
+[profiler]
+# If False fully disable profiling feature.
+#enabled = True
+
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True

etc/glance-registry-paste.ini

@@ -1,16 +1,16 @@
 # Use this pipeline for no auth - DEFAULT
 [pipeline:glance-registry]
-pipeline = unauthenticated-context registryapp
+pipeline = unauthenticated-context osprofiler registryapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-registry-keystone]
-pipeline = authtoken context registryapp
+pipeline = authtoken context osprofiler registryapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-registry-trusted-auth]
-pipeline = context registryapp
+pipeline = context osprofiler registryapp
 
 [app:registryapp]
 paste.app_factory = glance.registry.api:API.factory
@@ -23,3 +23,8 @@ paste.filter_factory = glance.api.middleware.context:UnauthenticatedContextMiddl
 
 [filter:authtoken]
 paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
+hmac_keys = SECRET_KEY
+enabled = yes

etc/glance-registry.conf

@@ -237,3 +237,10 @@ admin_password = %SERVICE_PASSWORD%
 # [pipeline:glance-registry-keystone], you would configure the flavor below
 # as 'keystone'.
 #flavor=
+
+[profiler]
+# If False fully disable profiling feature.
+#enabled = True
+
+# If False doesn't trace SQL requests.
+#trace_sqlalchemy = True

glance/cmd/api.py

@@ -39,12 +39,20 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
 if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
     sys.path.insert(0, possible_topdir)
 
+from oslo.config import cfg
+import osprofiler.notifier
+import osprofiler.web
+
 from glance.common import config
 from glance.common import exception
 from glance.common import wsgi
+from glance import notifier
 from glance.openstack.common import log
 import glance.store
 
+CONF = cfg.CONF
+CONF.import_group("profiler", "glance.common.wsgi")
+
 
 def fail(returncode, e):
     sys.stderr.write("ERROR: %s\n" % utils.exception_to_str(e))
@@ -60,6 +68,16 @@ def main():
         glance.store.create_stores()
         glance.store.verify_default_store()
 
+        if cfg.CONF.profiler.enabled:
+            _notifier = osprofiler.notifier.create("Messaging",
+                                                   notifier.messaging, {},
+                                                   notifier.get_transport(),
+                                                   "glance", "api",
+                                                   cfg.CONF.bind_host)
+            osprofiler.notifier.set(_notifier)
+        else:
+            osprofiler.web.disable()
+
         server = wsgi.Server()
         server.start(config.load_paste_app('glance-api'), default_port=9292)
         server.wait()

glance/cmd/registry.py

@@ -36,10 +36,18 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
 if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
     sys.path.insert(0, possible_topdir)
 
+from oslo.config import cfg
+import osprofiler.notifier
+import osprofiler.web
+
 from glance.common import config
 from glance.common import wsgi
+from glance import notifier
 from glance.openstack.common import log
 
+CONF = cfg.CONF
+CONF.import_group("profiler", "glance.common.wsgi")
+
 
 def main():
     try:
@@ -47,6 +55,17 @@ def main():
         wsgi.set_eventlet_hub()
         log.setup('glance')
 
+        if cfg.CONF.profiler.enabled:
+            _notifier = osprofiler.notifier.create("Messaging",
+                                                   notifier.messaging, {},
+                                                   notifier.get_transport(),
+                                                   "glance", "registry",
+                                                   cfg.CONF.bind_host)
+            osprofiler.notifier.set(_notifier)
+
+        else:
+            osprofiler.web.disable()
+
         server = wsgi.Server()
         server.start(config.load_paste_app('glance-registry'),
                      default_port=9191)

glance/common/client.py

@@ -32,6 +32,8 @@ except ImportError:
     import socket
     import ssl
 
+import osprofiler.web
+
 try:
     import sendfile  # noqa
     SENDFILE_SUPPORTED = True
@@ -457,6 +459,7 @@ class BaseClient(object):
         try:
             connection_type = self.get_connection_type()
             headers = self._encode_headers(headers or {})
+            headers.update(osprofiler.web.get_trace_id_headers())
 
             if 'x-auth-token' not in headers and self.auth_tok:
                 headers['x-auth-token'] = self.auth_tok

glance/common/wsgi.py

@@ -91,11 +91,19 @@ eventlet_opts = [
                       'Keystone v3 API with big service catalogs')),
 ]
 
+profiler_opts = [
+    cfg.BoolOpt("enabled", default=True,
+                help=_('If False fully disable profiling feature.')),
+    cfg.BoolOpt("trace_sqlalchemy", default=True,
+                help=_("If False doesn't trace SQL requests."))
+]
+
 
 CONF = cfg.CONF
 CONF.register_opts(bind_opts)
 CONF.register_opts(socket_opts)
 CONF.register_opts(eventlet_opts)
+CONF.register_opts(profiler_opts, group="profiler")
 
 
 def get_bind_addr(default_port=None):

glance/db/sqlalchemy/api.py

@@ -25,6 +25,7 @@ import threading
 from oslo.config import cfg
 from oslo.db import exception as db_exception
 from oslo.db.sqlalchemy import session
+import osprofiler.sqlalchemy
 from retrying import retry
 import six
 from six.moves import xrange
@@ -51,6 +52,7 @@ STATUSES = ['active', 'saving', 'queued', 'killed', 'pending_delete',
 
 CONF = cfg.CONF
 CONF.import_opt('debug', 'glance.openstack.common.log')
+CONF.import_group("profiler", "glance.common.wsgi")
 
 _FACADE = None
 _LOCK = threading.Lock()
@@ -71,6 +73,11 @@ def _create_facade_lazily():
         with _LOCK:
             if _FACADE is None:
                 _FACADE = session.EngineFacade.from_config(CONF)
+
+                if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
+                    osprofiler.sqlalchemy.add_tracing(sqlalchemy,
+                                                      _FACADE.get_engine(),
+                                                      "db")
     return _FACADE
 
 

glance/notifier.py

@@ -42,12 +42,16 @@ _ALIASES = {
 }
 
 
+def get_transport():
+    return messaging.get_transport(CONF, aliases=_ALIASES)
+
+
 class Notifier(object):
     """Uses a notification strategy to send out messages about events."""
 
     def __init__(self):
         publisher_id = CONF.default_publisher_id
-        self._transport = messaging.get_transport(CONF, aliases=_ALIASES)
+        self._transport = get_transport()
         self._notifier = messaging.Notifier(self._transport,
                                             publisher_id=publisher_id)
 

requirements.txt

@@ -49,3 +49,4 @@ oslo.i18n>=0.1.0  # Apache-2.0
 oslo.messaging>=1.4.0.0a3
 
 retrying>=1.2.2 # Apache-2.0
+osprofiler>=0.1.1