Merge "Provide the bones of profiler: api and middleware"

doc/source/topics/settings.rst

@@ -1662,6 +1662,42 @@ Ignore all listed Nova extensions, and behave as if they were unsupported.
 Can be used to selectively disable certain costly extensions for performance
 reasons.
 
+``OPENSTACK_PROFILER``
+----------------------
+
+.. versionadded:: 11.0.0(Ocata)
+
+Default: ``{"enabled": False}``
+
+Various settings related to integration with osprofiler library. Since it is a
+developer feature, it starts as disabled. To enable it, more than a single
+``"enabled"`` key should be specified. Additional keys that should be specified
+in that dictionary are:
+
+* ``"keys"`` is a list of strings, which are secret keys used to encode/decode
+  the profiler data contained in request headers. Encryption is used for security
+  purposes, other OpenStack components that are expected to profile themselves
+  with osprofiler using the data from the request that Horizon initiated must
+  share a common set of keys with the ones in Horizon config. List of keys is
+  used so that security keys could be changed in non-obtrusive manner for every
+  component in the cloud. Example: ``"keys": ["SECRET_KEY", "MORE_SECRET_KEY"]``.
+  For more details see `osprofiler documentation`_.
+* ``"notifier_connection_string"`` is a url to which trace messages are sent by
+  Horizon. For other components it is usually the only URL specified in config,
+  because other components act mostly as traces producers. Example:
+  ``"notifier_connection_string": "mongodb://%s' % OPENSTACK_HOST"``.
+* ``"receiver_connection_string"`` is a url from which traces are retrieved by
+  Horizon, needed because Horizon is not only the traces producer, but also a
+  consumer. Having 2 settings which usually contain the same value is legacy
+  feature from older versions of osprofiler when OpenStack components could use
+  oslo.messaging for notifications and the trace client used ceilometer as a
+  receiver backend. By default Horizon uses the same URL pointing to a MongoDB
+  cluster for both purposes, since ceilometer was too slow for using with UI.
+  Example: ``"receiver_connection_string": "mongodb://%s" % OPENSTACK_HOST``.
+
+.. _osprofiler documentation: http://docs.openstack.org/developer/osprofiler/integration.html#how-to-initialize-profiler-to-get-one-trace-across-all-services
+
+
 ``ALLOWED_PRIVATE_SUBNET_CIDR``
 -------------------------------
 

openstack_dashboard/contrib/developer/profiler/api.py

@@ -0,0 +1,99 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import contextlib
+
+from django.conf import settings
+from osprofiler.drivers.base import get_driver as profiler_get_driver
+from osprofiler import notifier
+from osprofiler import profiler
+from six.moves.urllib.parse import urlparse
+
+
+PROFILER_SETTINGS = getattr(settings, 'OPENSTACK_PROFILER', {})
+
+
+def init_notifier(connection_str, host="localhost"):
+    _notifier = notifier.create(
+        connection_str, project='horizon', service='horizon', host=host)
+    notifier.set(_notifier)
+
+
+@contextlib.contextmanager
+def traced(request, name, info=None):
+    if info is None:
+        info = {}
+    profiler_instance = profiler.get()
+    if profiler_instance is not None:
+        trace_id = profiler_instance.get_base_id()
+        info['user_id'] = request.user.id
+        with profiler.Trace(name, info=info):
+            yield trace_id
+    else:
+        yield
+
+
+def _get_engine_kwargs(request, connection_str):
+    from openstack_dashboard.api import base
+    engines_kwargs = {
+        # NOTE(tsufiev): actually Horizon doesn't use ceilometer backend (too
+        # slow for UI), but since osprofiler still supports it (due to API
+        # deprecation cycle limitations), Horizon also should support this
+        # option
+        'ceilometer': lambda req: {
+            'endpoint': base.url_for(req, 'metering'),
+            'insecure': getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False),
+            'cacert': getattr(settings, 'OPENSTACK_SSL_CACERT', None),
+            'token': (lambda: req.user.token.id),
+            'ceilometer_api_version': '2'
+        }
+    }
+
+    engine = urlparse(connection_str).scheme
+    return engines_kwargs.get(engine, lambda req: {})(request)
+
+
+def _get_engine(request):
+    connection_str = PROFILER_SETTINGS.get(
+        'receiver_connection_string', "mongodb://")
+    kwargs = _get_engine_kwargs(request, connection_str)
+    return profiler_get_driver(connection_str, **kwargs)
+
+
+def list_traces(request):
+    engine = _get_engine(request)
+    query = {"info.user_id": request.user.id}
+    fields = ['base_id', 'timestamp', 'info.request.path']
+    traces = engine.list_traces(query, fields)
+    return [{'id': trace['base_id'],
+             'timestamp': trace['timestamp'],
+             'origin': trace['info']['request']['path']} for trace in traces]
+
+
+def get_trace(request, trace_id):
+    def rec(_data, level=0):
+        _data['level'] = level
+        _data['is_leaf'] = not len(_data['children'])
+        _data['visible'] = True
+        _data['childrenVisible'] = True
+        for child in _data['children']:
+            rec(child, level + 1)
+        return _data
+
+    engine = _get_engine(request)
+    trace = engine.get_report(trace_id)
+    # throw away toplevel node which is dummy and doesn't contain any info,
+    # use its first and only child as the toplevel node
+    return rec(trace['children'][0])

openstack_dashboard/contrib/developer/profiler/middleware.py

@@ -0,0 +1,118 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from django.conf import settings
+from django.core import exceptions
+from django.core.urlresolvers import reverse
+from django.utils import safestring
+from django.utils.translation import ugettext_lazy as _
+from osprofiler import _utils as profiler_utils
+from osprofiler import profiler
+from osprofiler import web
+import six
+
+from horizon import messages
+from openstack_dashboard.contrib.developer.profiler import api
+
+_REQUIRED_KEYS = ("base_id", "hmac_key")
+_OPTIONAL_KEYS = ("parent_id",)
+
+PROFILER_SETTINGS = getattr(settings, 'OPENSTACK_PROFILER', {})
+
+
+class ProfilerClientMiddleware(object):
+    def process_request(self, request):
+        if 'profile_page' in request.COOKIES:
+            hmac_key = PROFILER_SETTINGS.get('keys')[0]
+            profiler.init(hmac_key)
+            for hdr_key, hdr_value in web.get_trace_id_headers().items():
+                request.META[hdr_key] = hdr_value
+        return None
+
+
+class ProfilerMiddleware(object):
+    def __init__(self):
+        self.name = PROFILER_SETTINGS.get('facility_name', 'horizon')
+        self.hmac_keys = PROFILER_SETTINGS.get('keys')
+        self._enabled = PROFILER_SETTINGS.get('enabled', False)
+        if self._enabled:
+            api.init_notifier(PROFILER_SETTINGS.get(
+                'notifier_connection_string', 'mongodb://'))
+        else:
+            raise exceptions.MiddlewareNotUsed()
+
+    @staticmethod
+    def is_authenticated(request):
+        return hasattr(request, "user") and request.user.is_authenticated()
+
+    def is_enabled(self, request):
+        return self.is_authenticated(request) and settings.DEBUG
+
+    @staticmethod
+    def _trace_is_valid(trace_info):
+        if not isinstance(trace_info, dict):
+            return False
+        trace_keys = set(six.iterkeys(trace_info))
+        if not all(k in trace_keys for k in _REQUIRED_KEYS):
+            return False
+        if trace_keys.difference(_REQUIRED_KEYS + _OPTIONAL_KEYS):
+            return False
+        return True
+
+    def process_view(self, request, view_func, view_args, view_kwargs):
+        # do not profile ajax requests for now
+        if not self.is_enabled(request) or request.is_ajax():
+            return None
+
+        trace_info = profiler_utils.signed_unpack(
+            request.META.get('X-Trace-Info'),
+            request.META.get('X-Trace-HMAC'),
+            self.hmac_keys)
+
+        if not self._trace_is_valid(trace_info):
+            return None
+
+        profiler.init(**trace_info)
+        info = {
+            'request': {
+                'path': request.path,
+                'query': request.GET.urlencode(),
+                'method': request.method,
+                'scheme': request.scheme
+            }
+        }
+        with api.traced(request, view_func.__name__, info) as trace_id:
+            response = view_func(request, *view_args, **view_kwargs)
+            url = reverse('horizon:developer:profiler:index')
+            message = safestring.mark_safe(
+                _('Traced with id %(id)s. Go to <a href="%(url)s">page</a>') %
+                {'id': trace_id, 'url': url})
+            messages.info(request, message)
+            return response
+
+    @staticmethod
+    def clear_profiling_cookies(request, response):
+        """Expire any cookie that initiated profiling request."""
+        if 'profile_page' in request.COOKIES:
+            path = request.path[:-1]
+            response.set_cookie('profile_page', max_age=0, path=path)
+
+    def process_response(self, request, response):
+        self.clear_profiling_cookies(request, response)
+        # do not profile ajax requests for now
+        if not self.is_enabled(request) or request.is_ajax():
+            return response
+
+        return response

openstack_dashboard/local/local_settings.d/_9030_profiler_settings.py.example

@@ -0,0 +1,6 @@
+OPENSTACK_PROFILER.update({
+    'enabled': True,
+    'keys': ['SECRET_KEY'],
+    'notifier_connection_string': 'mongodb://%s' % OPENSTACK_HOST,
+    'receiver_connection_string': 'mongodb://%s' % OPENSTACK_HOST
+})

openstack_dashboard/settings.py

@@ -113,6 +113,10 @@ MIDDLEWARE_CLASSES = (
     'horizon.themes.ThemeMiddleware',
     'django.middleware.locale.LocaleMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'openstack_dashboard.contrib.developer.profiler.middleware.'
+    'ProfilerClientMiddleware',
+    'openstack_dashboard.contrib.developer.profiler.middleware.'
+    'ProfilerMiddleware',
 )
 
 CACHED_TEMPLATE_LOADERS = [
@@ -319,6 +323,10 @@ ANGULAR_FEATURES = {
 # Notice all customizable configurations should be above this line
 XSTATIC_MODULES = settings_utils.BASE_XSTATIC_MODULES
 
+OPENSTACK_PROFILER = {
+    'enabled': False
+}
+
 try:
     from local.local_settings import *  # noqa
 except ImportError:

requirements.txt

@@ -24,6 +24,8 @@ oslo.i18n>=2.1.0 # Apache-2.0
 oslo.policy>=1.15.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
 oslo.utils>=3.18.0 # Apache-2.0
+osprofiler>=1.4.0 # Apache-2.0
+pymongo>=3.0.2,!=3.1  # Apache-2.0
 pyScss!=1.3.5,>=1.3.4 # MIT License
 python-ceilometerclient>=2.5.0 # Apache-2.0
 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0