openstack/manila
Code Issues Proposed changes

Merge "Integrate OSprofiler and Manila"

Browse Source
This commit is contained in:
Zuul 2021-02-25 17:16:55 +00:00 committed by Gerrit Code Review
commit ba6a2800d0
10 changed files with 114 additions and 10 deletions

17
etc/manila/api-paste.ini

@ -11,16 +11,16 @@ use = call:manila.api:root_app_factory
[composite:openstack_share_api]
use = call:manila.api.middleware.auth:pipeline_factory
noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth api
keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext api
noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth api
keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext api
[composite:openstack_share_api_v2]
use = call:manila.api.middleware.auth:pipeline_factory
noauth = cors faultwrap http_proxy_to_wsgi sizelimit noauth apiv2
noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit noauthv2 apiv2
keystone = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit authtoken keystonecontext apiv2
noauth = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauth apiv2
noauthv2 = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler noauthv2 apiv2
keystone = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
keystone_nolimit = cors faultwrap http_proxy_to_wsgi sizelimit osprofiler authtoken keystonecontext apiv2
[filter:faultwrap]
paste.filter_factory = manila.api.middleware.fault:FaultWrapper.factory
@ -34,6 +34,9 @@ paste.filter_factory = manila.api.middleware.auth:NoAuthMiddlewarev2_60.factory
[filter:sizelimit]
paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory
[filter:osprofiler]
paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
[filter:http_proxy_to_wsgi]
paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory

1
lower-constraints.txt

@ -72,6 +72,7 @@ oslo.service==2.4.0
oslo.upgradecheck==1.3.0
oslo.utils==4.7.0
oslotest==4.4.1
osprofiler==3.4.0
packaging==20.4
paramiko==2.7.2
Paste==3.4.3

19
manila/db/sqlalchemy/api.py

@ -23,6 +23,7 @@ import datetime
from functools import wraps
import ipaddress
import sys
import threading
import warnings
# NOTE(uglide): Required to override default oslo_db Query class
@ -37,8 +38,10 @@ from oslo_db.sqlalchemy import session
from oslo_db.sqlalchemy import utils as db_utils
from oslo_log import log
from oslo_utils import excutils
from oslo_utils import importutils
from oslo_utils import timeutils
from oslo_utils import uuidutils
import sqlalchemy
from sqlalchemy import MetaData
from sqlalchemy import or_
from sqlalchemy.orm import joinedload
@ -54,7 +57,11 @@ from manila import exception
from manila.i18n import _
from manila import quota
osprofiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
CONF = cfg.CONF
CONF.import_group("profiler", "manila.service")
LOG = log.getLogger(__name__)
QUOTAS = quota.QUOTAS
@ -63,6 +70,8 @@ _DEFAULT_QUOTA_NAME = 'default'
PER_PROJECT_QUOTAS = []
_FACADE = None
_LOCK = threading.Lock()
_DEFAULT_SQL_CONNECTION = 'sqlite://'
db_options.set_defaults(cfg.CONF,
@ -70,9 +79,15 @@ db_options.set_defaults(cfg.CONF,
def _create_facade_lazily():
global _FACADE
global _LOCK, _FACADE
if _FACADE is None:
_FACADE = session.EngineFacade.from_config(cfg.CONF)
with _LOCK:
if _FACADE is None:
_FACADE = session.EngineFacade.from_config(cfg.CONF)
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
osprofiler_sqlalchemy.add_tracing(sqlalchemy,
_FACADE.get_engine(),
"db")
return _FACADE

19
manila/rpc.py

@ -29,6 +29,8 @@ from oslo_config import cfg
import oslo_messaging as messaging
from oslo_messaging.rpc import dispatcher
from oslo_serialization import jsonutils
from oslo_utils import importutils
profiler = importutils.try_import('osprofiler.profiler')
import manila.context
import manila.exception
@ -115,9 +117,24 @@ class RequestContextSerializer(messaging.Serializer):
return self._base.deserialize_entity(context, entity)
def serialize_context(self, context):
return context.to_dict()
_context = context.to_dict()
if profiler is not None:
prof = profiler.get()
if prof:
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
_context.update({"trace_info": trace_info})
return _context
def deserialize_context(self, context):
trace_info = context.pop("trace_info", None)
if trace_info:
if profiler is not None:
profiler.init(**trace_info)
return manila.context.RequestContext.from_dict(context)

32
manila/service.py

@ -36,6 +36,10 @@ from manila import exception
from manila import rpc
from manila import version
osprofiler_initializer = importutils.try_import('osprofiler.initializer')
profiler = importutils.try_import('osprofiler.profiler')
profiler_opts = importutils.try_import('osprofiler.opts')
LOG = log.getLogger(__name__)
service_opts = [
@ -68,6 +72,26 @@ service_opts = [
CONF = cfg.CONF
CONF.register_opts(service_opts)
if profiler_opts:
profiler_opts.set_defaults(CONF)
def setup_profiler(binary, host):
if (osprofiler_initializer is None or
profiler is None or
profiler_opts is None):
LOG.debug('osprofiler is not present')
return
if CONF.profiler.enabled:
osprofiler_initializer.init_from_conf(
conf=CONF,
context=context.get_admin_context().to_dict(),
project="manila",
service=binary,
host=host
)
LOG.warning("OSProfiler is enabled.")
class Service(service.Service):
@ -89,6 +113,10 @@ class Service(service.Service):
self.topic = topic
self.manager_class_name = manager
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled and profiler is not None:
manager_class = profiler.trace_cls("rpc")(manager_class)
self.service = None
self.manager = manager_class(host=self.host,
service_name=service_name,
*args, **kwargs)
@ -100,6 +128,8 @@ class Service(service.Service):
self.timers = []
self.coordinator = coordination
setup_profiler(binary, host)
def start(self):
version_string = version.version_string()
LOG.info('Starting %(topic)s node (version %(version_string)s)',
@ -311,6 +341,8 @@ class WSGIService(service.ServiceBase):
"greater than 1. Input value ignored.", {'name': name})
# Reset workers to default
self.workers = None
setup_profiler(name, self.host)
self.server = wsgi.Server(
CONF,
name,

4
manila/share/manager.py

@ -55,6 +55,8 @@ from manila.share import snapshot_access
from manila.share import utils as share_utils
from manila import utils
profiler = importutils.try_import('osprofiler.profiler')
LOG = log.getLogger(__name__)
share_manager_opts = [
@ -259,6 +261,8 @@ class ShareManager(manager.SchedulerDependentManager):
self.message_api = message_api.API()
self.share_api = api.API()
if CONF.profiler.enabled and profiler is not None:
self.driver = profiler.trace_cls("driver")(self.driver)
self.hooks = []
self._init_hook_drivers()

15
manila/test.py

@ -153,12 +153,27 @@ class TestCase(base_test.BaseTestCase):
fake_notifier.stub_notifier(self)
self._disable_osprofiler()
# Locks must be cleaned up after tests
CONF.set_override('backend_url', 'file://' + lock_path,
group='coordination')
coordination.LOCK_COORDINATOR.start()
self.addCleanup(coordination.LOCK_COORDINATOR.stop)
def _disable_osprofiler(self):
"""Disable osprofiler.
osprofiler should not run for unit tests.
"""
def side_effect(value):
return value
mock_decorator = mock.MagicMock(side_effect=side_effect)
p = mock.patch("osprofiler.profiler.trace_cls",
return_value=mock_decorator)
p.start()
def tearDown(self):
"""Runs after each test method to tear down test environment."""
super(TestCase, self).tearDown()

2
manila/wsgi/wsgi.py

@ -22,6 +22,7 @@ from oslo_service import wsgi
# Need to register global_opts
from manila.common import config
from manila import rpc
from manila import service
from manila import version
CONF = cfg.CONF
@ -33,4 +34,5 @@ def initialize_application():
config.verify_share_protocols()
log.setup(CONF, "manila")
rpc.init(CONF)
service.setup_profiler("manila-api", CONF.host)
return wsgi.Loader(CONF).load_app(name='osapi_share')

14
releasenotes/notes/bp-integrate-os-profiler-b637041861029175.yaml Normal file

@ -0,0 +1,14 @@
---
features:
- |
OSprofiler support was introduced. To allow its usage the api-paste.ini
file needs to be modified to contain osprofiler middleware. Also
`[profiler]` section needs to be added to the manila.conf file with
`enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined.
security:
- OSprofiler support requires passing of trace information
between various OpenStack services. This information is
securely signed by one of HMAC keys, defined in manila.conf configuration
file. To allow cross-project tracing user should use the key, that is
common among all OpenStack services they want to trace.

1
requirements.txt

@ -25,6 +25,7 @@ oslo.service>=2.4.0 # Apache-2.0
oslo.upgradecheck>=1.3.0 # Apache-2.0
oslo.utils>=4.7.0 # Apache-2.0
oslo.concurrency>=4.3.0 # Apache-2.0
osprofiler>=3.4.0 # Apache-2.0
paramiko>=2.7.2 # LGPLv2.1+
Paste>=3.4.3 # MIT
PasteDeploy>=2.1.0 # MIT