Add OSprofiler support

* Add osprofiler wsgi middleware. This middleware is used for 2 things:
  1) It checks that person who wants to trace is trusted and knows
     secret HMAC key.
  2) It starts tracing in case of proper trace headers
     and adds first wsgi trace point, with info about HTTP request

* Add initialization of osprofiler at start of service
  Currently that includes oslo.messaging notifer instance creation
  to send Ceilometer backend notifications.

Neutron client change: Ic11796889075b2a0e589b70398fc4d4ed6f3ef7c

Co-authored-by: Ryan Moats <rmoats@us.ibm.com>
Depends-On: I5102eb46a7a377eca31375a0d64951ba1fdd035d
Closes-Bug: #1335640
DocImpact Add devref and operator documentation on how to use this
APIImpact
Change-Id: I7fa2ad57dc5763ce72cba6945ebcadef2188e8bd

etc/api-paste.ini

@@ -32,3 +32,6 @@ paste.app_factory = neutron.api.versions:Versions.factory
 
 [app:neutronapiapp_v2_0]
 paste.app_factory = neutron.api.v2.router:APIRouter.factory
+
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory

neutron/common/profiler.py

@@ -0,0 +1,48 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+import oslo_messaging
+import osprofiler.notifier
+from osprofiler import opts as profiler_opts
+import osprofiler.web
+
+from neutron._i18n import _LI
+
+CONF = cfg.CONF
+profiler_opts.set_defaults(CONF)
+LOG = logging.getLogger(__name__)
+
+
+def setup(name, host='0.0.0.0'):  # nosec
+    """Setup OSprofiler notifier and enable profiling.
+
+    :param name: name of the service, that will be profiled
+    :param host: host (either host name or host address) the service will be
+                 running on. By default host will be set to 0.0.0.0, but more
+                 specified host name / address usage is highly recommended.
+    """
+    if CONF.profiler.enabled:
+        _notifier = osprofiler.notifier.create(
+            "Messaging", oslo_messaging, {},
+            oslo_messaging.get_transport(CONF), "neutron", name, host)
+        osprofiler.notifier.set(_notifier)
+        osprofiler.web.enable(CONF.profiler.hmac_keys)
+        LOG.info(_LI("OSProfiler is enabled.\n"
+                     "Traces provided from the profiler "
+                     "can only be subscribed to using the same HMAC keys that "
+                     "are configured in Neutron's configuration file "
+                     "under the [profiler] section.\n To disable OSprofiler "
+                     "set in /etc/neutron/neutron.conf:\n"
+                     "[profiler]\n"
+                     "enabled=false"))

neutron/common/rpc.py

@@ -25,6 +25,7 @@ import oslo_messaging
 from oslo_messaging import serializer as om_serializer
 from oslo_service import service
 from oslo_utils import excutils
+from osprofiler import profiler
 
 from neutron._i18n import _LE, _LW
 from neutron.common import exceptions
@@ -212,10 +213,22 @@ class RequestContextSerializer(om_serializer.Serializer):
         return self._base.deserialize_entity(ctxt, entity)
 
     def serialize_context(self, ctxt):
-        return ctxt.to_dict()
+        _context = ctxt.to_dict()
+        prof = profiler.get()
+        if prof:
+            trace_info = {
+                "hmac_key": prof.hmac_key,
+                "base_id": prof.get_base_id(),
+                "parent_id": prof.get_id()
+            }
+            _context['trace_info'] = trace_info
+        return _context
 
     def deserialize_context(self, ctxt):
         rpc_ctxt_dict = ctxt.copy()
+        trace_info = rpc_ctxt_dict.pop("trace_info", None)
+        if trace_info:
+            profiler.init(**trace_info)
         user_id = rpc_ctxt_dict.pop('user_id', None)
         if not user_id:
             user_id = rpc_ctxt_dict.pop('user', None)
@@ -225,6 +238,7 @@ class RequestContextSerializer(om_serializer.Serializer):
         return context.Context(user_id, tenant_id, **rpc_ctxt_dict)
 
 
+@profiler.trace_cls("rpc")
 class Service(service.Service):
     """Service object for binaries running on hosts.
 

neutron/db/api.py

@@ -23,6 +23,8 @@ from oslo_db import exception as db_exc
 from oslo_db.sqlalchemy import enginefacade
 from oslo_utils import excutils
 from oslo_utils import uuidutils
+import osprofiler.sqlalchemy
+import sqlalchemy
 
 from neutron.db import common_db_mixin
 
@@ -58,6 +60,11 @@ def _create_facade_lazily():
         context_manager.configure(sqlite_fk=True, **cfg.CONF.database)
         _FACADE = context_manager._factory.get_legacy_facade()
 
+        if cfg.CONF.profiler.enabled and cfg.CONF.profiler.trace_sqlalchemy:
+            osprofiler.sqlalchemy.add_tracing(sqlalchemy,
+                                              _FACADE.get_engine(),
+                                              "db")
+
     return _FACADE
 
 

neutron/manager.py

@@ -19,6 +19,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging
 from oslo_service import periodic_task
+from osprofiler import profiler
 import six
 
 from neutron._i18n import _, _LI
@@ -31,7 +32,13 @@ LOG = logging.getLogger(__name__)
 CORE_PLUGINS_NAMESPACE = 'neutron.core_plugins'
 
 
+class ManagerMeta(profiler.TracedMeta, type(periodic_task.PeriodicTasks)):
+    pass
+
+
+@six.add_metaclass(ManagerMeta)
 class Manager(periodic_task.PeriodicTasks):
+    __trace_args__ = {"name": "rpc"}
 
     # Set RPC API version to 1.0 by default.
     target = oslo_messaging.Target(version='1.0')
@@ -86,6 +93,7 @@ def validate_pre_plugin_load():
         return msg
 
 
+@six.add_metaclass(profiler.TracedMeta)
 class NeutronManager(object):
     """Neutron's Manager class.
 
@@ -95,6 +103,7 @@ class NeutronManager(object):
     The caller should make sure that NeutronManager is a singleton.
     """
     _instance = None
+    __trace_args__ = {"name": "rpc"}
 
     def __init__(self, options=None, config_file=None):
         # If no options have been provided, create an empty dict

neutron/pecan_wsgi/app.py

@@ -88,10 +88,12 @@ def _wrap_app(app):
     app.set_latent(
         allow_headers=['X-Auth-Token', 'X-Identity-Status', 'X-Roles',
                        'X-Service-Catalog', 'X-User-Id', 'X-Tenant-Id',
-                       'X-OpenStack-Request-ID'],
+                       'X-OpenStack-Request-ID',
+                       'X-Trace-Info', 'X-Trace-HMAC'],
         allow_methods=['GET', 'PUT', 'POST', 'DELETE', 'PATCH'],
         expose_headers=['X-Auth-Token', 'X-Subject-Token', 'X-Service-Token',
-                        'X-OpenStack-Request-ID']
+                        'X-OpenStack-Request-ID',
+                        'X-Trace-Info', 'X-Trace-HMAC']
     )
 
     return app

neutron/plugins/ml2/drivers/agent/_common_agent.py

@@ -23,6 +23,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_service import service
+from osprofiler import profiler
 
 from neutron._i18n import _LE, _LI
 from neutron.agent.l2.extensions import manager as ext_manager
@@ -39,6 +40,7 @@ from neutron.plugins.ml2.drivers.agent import config as cagt_config  # noqa
 LOG = logging.getLogger(__name__)
 
 
+@profiler.trace_cls("rpc")
 class CommonAgentLoop(service.Service):
 
     def __init__(self, manager, polling_interval,

neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py

@@ -37,6 +37,7 @@ from neutron.agent.linux import utils
 from neutron.agent import securitygroups_rpc as sg_rpc
 from neutron.common import config as common_config
 from neutron.common import exceptions
+from neutron.common import profiler as setup_profiler
 from neutron.common import topics
 from neutron.common import utils as n_utils
 from neutron.plugins.common import constants as p_const
@@ -928,6 +929,7 @@ def main():
     agent = ca.CommonAgentLoop(manager, polling_interval, quitting_rpc_timeout,
                                constants.AGENT_TYPE_LINUXBRIDGE,
                                LB_AGENT_BINARY)
+    setup_profiler.setup("neutron-linuxbridge-agent", cfg.CONF.host)
     LOG.info(_LI("Agent initialized successfully, now running... "))
     launcher = service.launch(cfg.CONF, agent)
     launcher.wait()

neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py

@@ -25,6 +25,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging
 from oslo_service import loopingcall
+from osprofiler import profiler
 import six
 
 from neutron._i18n import _, _LE, _LI, _LW
@@ -33,6 +34,7 @@ from neutron.agent import rpc as agent_rpc
 from neutron.agent import securitygroups_rpc as sg_rpc
 from neutron.api.rpc.callbacks import resources
 from neutron.common import config as common_config
+from neutron.common import profiler as setup_profiler
 from neutron.common import topics
 from neutron.common import utils as n_utils
 from neutron import context
@@ -103,6 +105,7 @@ class SriovNicSwitchRpcCallbacks(sg_rpc.SecurityGroupAgentRpcCallbackMixin):
             self.agent.updated_devices.add(port_data['device'])
 
 
+@profiler.trace_cls("rpc")
 class SriovNicSwitchAgent(object):
     def __init__(self, physical_devices_mappings, exclude_devices,
                  polling_interval):
@@ -458,5 +461,6 @@ def main():
         LOG.exception(_LE("Agent Initialization Failed"))
         raise SystemExit(1)
     # Start everything.
+    setup_profiler.setup("neutron-sriov-nic-agent", cfg.CONF.host)
     LOG.info(_LI("Agent initialized successfully, now running... "))
     agent.daemon_loop()

neutron/plugins/ml2/drivers/openvswitch/agent/main.py

@@ -22,6 +22,7 @@ from oslo_log import log as logging
 from oslo_utils import importutils
 
 from neutron.common import config as common_config
+from neutron.common import profiler
 from neutron.common import utils as n_utils
 
 
@@ -46,4 +47,5 @@ def main():
     mod.init_config()
     common_config.setup_logging()
     n_utils.log_opt_values(LOG)
+    profiler.setup("neutron-ovs-agent", cfg.CONF.host)
     mod.main()

neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py

@@ -20,6 +20,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging
 from oslo_utils import excutils
+from osprofiler import profiler
 
 from neutron._i18n import _LE, _LI, _LW
 from neutron.common import utils as n_utils
@@ -112,6 +113,7 @@ class OVSPort(object):
         return self.ofport
 
 
+@profiler.trace_cls("ovs_dvr_agent")
 class OVSDVRNeutronAgent(object):
     '''
     Implements OVS-based DVR(Distributed Virtual Router), for overlay networks.

neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py

@@ -28,6 +28,7 @@ from oslo_log import log as logging
 import oslo_messaging
 from oslo_service import loopingcall
 from oslo_service import systemd
+from osprofiler import profiler
 import six
 from six import moves
 
@@ -98,6 +99,7 @@ def has_zero_prefixlen_address(ip_addresses):
     return any(netaddr.IPNetwork(ip).prefixlen == 0 for ip in ip_addresses)
 
 
+@profiler.trace_cls("rpc")
 class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
                       l2population_rpc.L2populationRpcCallBackTunnelMixin,
                       dvr_rpc.DVRAgentRpcCallbackMixin):

neutron/service.py

@@ -28,6 +28,7 @@ from oslo_utils import importutils
 
 from neutron._i18n import _LE, _LI
 from neutron.common import config
+from neutron.common import profiler
 from neutron.common import rpc as n_rpc
 from neutron.conf import service
 from neutron import context
@@ -64,6 +65,9 @@ class WsgiService(object):
 
 class NeutronApiService(WsgiService):
     """Class for neutron-api service."""
+    def __init__(self, app_name):
+        profiler.setup('neutron-server', cfg.CONF.host)
+        super(NeutronApiService, self).__init__(app_name)
 
     @classmethod
     def create(cls, app_name='neutron'):
@@ -246,6 +250,7 @@ class Service(n_rpc.Service):
         self.periodic_fuzzy_delay = periodic_fuzzy_delay
         self.saved_args, self.saved_kwargs = args, kwargs
         self.timers = []
+        profiler.setup(binary, host)
         super(Service, self).__init__(host, topic, manager=self.manager)
 
     def start(self):

neutron/tests/contrib/gate_hook.sh

@@ -61,6 +61,7 @@ case $VENV in
 "api"|"api-pecan"|"full-pecan"|"dsvm-scenario")
     load_rc_hook api_extensions
     load_rc_hook qos
+    load_conf_hook osprofiler
     if [[ "$VENV" =~ "pecan" ]]; then
         load_conf_hook pecan
     fi

neutron/tests/contrib/hooks/osprofiler

@@ -0,0 +1,7 @@
+[[post-config|/etc/neutron/api-paste.ini]]
+
+[composite:neutronapi_v2_0]
+use = call:neutron.auth:pipeline_factory
+noauth = cors request_id catch_errors osprofiler extensions neutronapiapp_v2_0
+keystone = cors request_id catch_errors osprofiler authtoken keystonecontext extensions neutronapiapp_v2_0
+

releasenotes/notes/add-osprofiler-support-7fc2de3001187075.yaml

@@ -0,0 +1,32 @@
+---
+fixes:
+  - Missing OSprofiler support was added. This cross-project profiling library
+    allows to trace various OpenStack requests through all OpenStack
+    services that support it. To initiate OpenStack
+    request tracing `--profile <HMAC_KEY>` option needs to be added to
+    the CLI command. This key needs to present one of the secret keys
+    defined in neutron.conf configuration file with `hmac_keys` option
+    under the `[profiler]` configuration section. To enable or disable
+    Neutron profiling the appropriate `enabled` option under the same
+    section needs to be set either to `True` or `False`. By default
+    Neutron will trace all API and RPC requests, but there is an opportunity
+    to trace DB requests as well. For this purpose `trace_sqlalchemy`
+    option needs to be set to `True`. As a prerequisite OSprofiler
+    library and its storage backend needs to be installed to the
+    environment. If so (and if profiling is enabled in neutron.conf)
+    the trace can be generated via command -
+    `$ neutron --profile SECRET_KEY <subcommand>`.
+    At the end of output there will be message with <trace_id>, and
+    to plot nice HTML graphs the following command should be used -
+    `$ osprofiler trace show <trace_id> --html --out result.html`
+upgrade:
+  - OSprofiler support was introduced. To allow its usage the api-paste.ini
+    file needs to be modified to contain osprofiler middleware. Also
+    `[profiler]` section needs to be added to the neutron.conf file with
+    `enabled`, `hmac_keys` and `trace_sqlalchemy` flags defined.
+security:
+  - OSprofiler support requires passing of trace information
+    between various OpenStack services. This information is
+    securely signed by one of HMAC keys, defined in neutron.conf configuration
+    file. To allow cross-project tracing user should use the key, that is
+    common among all OpenStack services he or she wants to trace.

requirements.txt

@@ -42,6 +42,7 @@ oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.10.0 # Apache-2.0
 oslo.utils>=3.5.0 # Apache-2.0
 oslo.versionedobjects!=1.9.0,>=1.5.0 # Apache-2.0
+osprofiler>=1.3.0 # Apache-2.0
 ovs>=2.5.0;python_version=='2.7' # Apache-2.0
 ovs>=2.6.0.dev1;python_version>='3.4' # Apache-2.0