Integrate OSprofiler and Manila

*) Add osprofiler wsgi middleware

This middleware is used for 2 things:
1) It checks that person who want to trace is trusted and knows
secret HMAC key.
2) It start tracing in case of proper trace headers and add
first wsgi trace point, with info about HTTP request
*) Add initialization of osprofiler at start of serivce.

You should use python-manilaclient with this patch:

https://review.opendev.org/#/c/769731

Run any command with --profile SECRET_KEY

  $ manila --profile SECRET_KEY create NFS 1 --name <share> \
    --description <share_description> --share-network <network> \
    --share-type default
  # it will print <Trace ID>

Get pretty HTML with traces:

  $ osprofiler trace show --html <Trace ID> --connection-string \
   <connection_string> --out <output.html>
  e.g. --connection-string can be redis://localhost:6379

Note that osprofiler should be run from admin user name & tenant.

Implements: blueprint manila-os-profiler
Change-Id: I3bce1f04d1cfebfacd78ed135a949a068c78987d

etc/manila/api-paste.ini

@@ -11,16 +11,16 @@ use = call:manila.api:root_app_factory
 
 [composite:openstack_share_api]
 use = call:manila.api.middleware.auth:pipeline_factory
-noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth api
-keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
-keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
+noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth api
+keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
+keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
 
 [composite:openstack_share_api_v2]
 use = call:manila.api.middleware.auth:pipeline_factory
-noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth apiv2
-noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit noauthv2 apiv2
-keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
-keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
+noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth apiv2
+noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauthv2 apiv2
+keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
+keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
 
 [filter:faultwrap]
 paste.filter_factory = manila.api.middleware.fault:FaultWrapper.factory
@@ -34,6 +34,9 @@ paste.filter_factory = manila.api.middleware.auth:NoAuthMiddlewarev2_60.factory
 [filter:sizelimit]
 paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory
 
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
+
 [filter:http_proxy_to_wsgi]
 paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory
 

lower-constraints.txt

@@ -72,6 +72,7 @@ oslo.service==2.4.0
 oslo.upgradecheck==1.3.0
 oslo.utils==4.7.0
 oslotest==4.4.1
+osprofiler==3.4.0
 packaging==20.4
 paramiko==2.7.2
 Paste==3.4.3

manila/db/sqlalchemy/api.py

@@ -23,6 +23,7 @@ import datetime
 from functools import wraps
 import ipaddress
 import sys
+import threading
 import warnings
 
 # NOTE(uglide): Required to override default oslo_db Query class
@@ -37,8 +38,10 @@ from oslo_db.sqlalchemy import session
 from oslo_db.sqlalchemy import utils as db_utils
 from oslo_log import log
 from oslo_utils import excutils
+from oslo_utils import importutils
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
+import sqlalchemy
 from sqlalchemy import MetaData
 from sqlalchemy import or_
 from sqlalchemy.orm import joinedload
@@ -54,7 +57,11 @@ from manila import exception
 from manila.i18n import _
 from manila import quota
 
+
+osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
+
 CONF = cfg.CONF
+CONF.import_group("profiler", "manila.service")
 
 LOG = log.getLogger(__name__)
 QUOTAS = quota.QUOTAS
@@ -63,6 +70,8 @@ _DEFAULT_QUOTA_NAME = 'default'
 PER_PROJECT_QUOTAS = []
 
 _FACADE = None
+_LOCK = threading.Lock()
+
 
 _DEFAULT_SQL_CONNECTION = 'sqlite://'
 db_options.set_defaults(cfg.CONF,
@@ -70,9 +79,15 @@ db_options.set_defaults(cfg.CONF,
 
 
 def _create_facade_lazily():
-    global _FACADE
+    global _LOCK, _FACADE
     if _FACADE is None:
-        _FACADE = session.EngineFacade.from_config(cfg.CONF)
+        with _LOCK:
+            if _FACADE is None:
+                _FACADE = session.EngineFacade.from_config(cfg.CONF)
+            if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
+                osprofiler_sqlalchemy.add_tracing(sqlalchemy,
+                                                  _FACADE.get_engine(),
+                                                  "db")
     return _FACADE
 
 

manila/rpc.py

@@ -29,6 +29,8 @@ from oslo_config import cfg
 import oslo_messaging as messaging
 from oslo_messaging.rpc import dispatcher
 from oslo_serialization import jsonutils
+from oslo_utils import importutils
+profiler = importutils.try_import('osprofiler.profiler')
 
 import manila.context
 import manila.exception
@@ -115,9 +117,24 @@ class RequestContextSerializer(messaging.Serializer):
         return self._base.deserialize_entity(context, entity)
 
     def serialize_context(self, context):
-        return context.to_dict()
+        _context = context.to_dict()
+        if profiler is not None:
+            prof = profiler.get()
+            if prof:
+                trace_info = {
+                    "hmac_key": prof.hmac_key,
+                    "base_id": prof.get_base_id(),
+                    "parent_id": prof.get_id()
+                }
+                _context.update({"trace_info": trace_info})
+        return _context
 
     def deserialize_context(self, context):
+        trace_info = context.pop("trace_info", None)
+        if trace_info:
+            if profiler is not None:
+                profiler.init(**trace_info)
+
         return manila.context.RequestContext.from_dict(context)
 
 

manila/service.py

@@ -36,6 +36,10 @@ from manila import exception
 from manila import rpc
 from manila import version
 
+osprofiler_initializer = importutils.try_import('osprofiler.initializer')
+profiler = importutils.try_import('osprofiler.profiler')
+profiler_opts = importutils.try_import('osprofiler.opts')
+
 LOG = log.getLogger(__name__)
 
 service_opts = [
@@ -68,6 +72,26 @@ service_opts = [
 
 CONF = cfg.CONF
 CONF.register_opts(service_opts)
+if profiler_opts:
+    profiler_opts.set_defaults(CONF)
+
+
+def setup_profiler(binary, host):
+    if (osprofiler_initializer is None or
+            profiler is None or
+            profiler_opts is None):
+        LOG.debug('osprofiler is not present')
+        return
+
+    if CONF.profiler.enabled:
+        osprofiler_initializer.init_from_conf(
+            conf=CONF,
+            context=context.get_admin_context().to_dict(),
+            project="manila",
+            service=binary,
+            host=host
+        )
+        LOG.warning("OSProfiler is enabled.")
 
 
 class Service(service.Service):
@@ -89,6 +113,10 @@ class Service(service.Service):
         self.topic = topic
         self.manager_class_name = manager
         manager_class = importutils.import_class(self.manager_class_name)
+        if CONF.profiler.enabled and profiler is not None:
+            manager_class = profiler.trace_cls("rpc")(manager_class)
+
+        self.service = None
         self.manager = manager_class(host=self.host,
                                      service_name=service_name,
                                      *args, **kwargs)
@@ -100,6 +128,8 @@ class Service(service.Service):
         self.timers = []
         self.coordinator = coordination
 
+        setup_profiler(binary, host)
+
     def start(self):
         version_string = version.version_string()
         LOG.info('Starting %(topic)s node (version %(version_string)s)',
@@ -311,6 +341,8 @@ class WSGIService(service.ServiceBase):
                 "greater than 1.  Input value ignored.", {'name': name})
             # Reset workers to default
             self.workers = None
+        setup_profiler(name, self.host)
+
         self.server = wsgi.Server(
             CONF,
             name,

manila/share/manager.py

@@ -55,6 +55,8 @@ from manila.share import snapshot_access
 from manila.share import utils as share_utils
 from manila import utils
 
+profiler = importutils.try_import('osprofiler.profiler')
+
 LOG = log.getLogger(__name__)
 
 share_manager_opts = [
@@ -259,6 +261,8 @@ class ShareManager(manager.SchedulerDependentManager):
 
         self.message_api = message_api.API()
         self.share_api = api.API()
+        if CONF.profiler.enabled and profiler is not None:
+            self.driver = profiler.trace_cls("driver")(self.driver)
         self.hooks = []
         self._init_hook_drivers()
 

manila/test.py

@@ -153,12 +153,27 @@ class TestCase(base_test.BaseTestCase):
 
         fake_notifier.stub_notifier(self)
 
+        self._disable_osprofiler()
+
         # Locks must be cleaned up after tests
         CONF.set_override('backend_url', 'file://' + lock_path,
                           group='coordination')
         coordination.LOCK_COORDINATOR.start()
         self.addCleanup(coordination.LOCK_COORDINATOR.stop)
 
+    def _disable_osprofiler(self):
+        """Disable osprofiler.
+
+        osprofiler should not run for unit tests.
+        """
+
+        def side_effect(value):
+            return value
+        mock_decorator = mock.MagicMock(side_effect=side_effect)
+        p = mock.patch("osprofiler.profiler.trace_cls",
+                       return_value=mock_decorator)
+        p.start()
+
     def tearDown(self):
         """Runs after each test method to tear down test environment."""
         super(TestCase, self).tearDown()

manila/wsgi/wsgi.py

@@ -22,6 +22,7 @@ from oslo_service import wsgi
 # Need to register global_opts
 from manila.common import config
 from manila import rpc
+from manila import service
 from manila import version
 
 CONF = cfg.CONF
@@ -33,4 +34,5 @@ def initialize_application():
     config.verify_share_protocols()
     log.setup(CONF, "manila")
     rpc.init(CONF)
+    service.setup_profiler("manila-api", CONF.host)
     return wsgi.Loader(CONF).load_app(name='osapi_share')

releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml

@@ -0,0 +1,14 @@
+---
+features:
+  - |
+    OSprofiler support was introduced. To allow its usage the api-paste.ini
+    file needs to be modified to contain osprofiler middleware. Also
+    `[profiler]` section needs to be added to the manila.conf file with
+    `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined.
+security:
+  - OSprofiler support requires passing of trace information
+    between various OpenStack services. This information is
+    securely signed by one of HMAC keys, defined in manila.conf configuration
+    file. To allow cross-project tracing user should use the key, that is
+    common among all OpenStack services they want to trace.
+

requirements.txt

@@ -25,6 +25,7 @@ oslo.service>=2.4.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0 # Apache-2.0
 oslo.utils>=4.7.0 # Apache-2.0
 oslo.concurrency>=4.3.0 # Apache-2.0
+osprofiler>=3.4.0 # Apache-2.0
 paramiko>=2.7.2 # LGPLv2.1+
 Paste>=3.4.3 # MIT
 PasteDeploy>=2.1.0 # MIT