Merge "NetApp ONTAP: Add support for filtering API tracing"

manila/share/drivers/netapp/dataontap/client/api.py

@@ -19,6 +19,7 @@ Contains classes required to issue API calls to Data ONTAP and OnCommand DFM.
 """
 
 import copy
+import re
 
 from lxml import etree
 from oslo_log import log
@@ -27,6 +28,7 @@ from six.moves import urllib
 
 from manila import exception
 from manila.i18n import _
+from manila.share.drivers.netapp import utils
 
 LOG = log.getLogger(__name__)
 
@@ -69,7 +71,8 @@ class NaServer(object):
     def __init__(self, host, server_type=SERVER_TYPE_FILER,
                  transport_type=TRANSPORT_TYPE_HTTP,
                  style=STYLE_LOGIN_PASSWORD, username=None,
-                 password=None, port=None, trace=False):
+                 password=None, port=None, trace=False,
+                 api_trace_pattern=utils.API_TRACE_PATTERN):
         self._host = host
         self.set_server_type(server_type)
         self.set_transport_type(transport_type)
@@ -79,8 +82,8 @@ class NaServer(object):
         self._username = username
         self._password = password
         self._trace = trace
+        self._api_trace_pattern = api_trace_pattern
         self._refresh_conn = True
-        self._trace = trace
 
         LOG.debug('Using NetApp controller: %s', self._host)
 
@@ -213,19 +216,18 @@ class NaServer(object):
         self._password = password
         self._refresh_conn = True
 
-    def set_trace(self, trace=True):
-        """Enable or disable the API tracing facility."""
-        self._trace = trace
-
     def invoke_elem(self, na_element, enable_tunneling=False):
         """Invoke the API on the server."""
         if na_element and not isinstance(na_element, NaElement):
             ValueError('NaElement must be supplied to invoke API')
-
         request, request_element = self._create_request(na_element,
                                                         enable_tunneling)
 
-        if self._trace:
+        api_name = na_element.get_name()
+        api_name_matches_regex = (re.match(self._api_trace_pattern, api_name)
+                                  is not None)
+
+        if self._trace and api_name_matches_regex:
             LOG.debug("Request: %s", request_element.to_string(pretty=True))
 
         if (not hasattr(self, '_opener') or not self._opener
@@ -246,7 +248,7 @@ class NaServer(object):
         response_xml = response.read()
         response_element = self._get_result(response_xml)
 
-        if self._trace:
+        if self._trace and api_name_matches_regex:
             LOG.debug("Response: %s", response_element.to_string(pretty=True))
 
         return response_element

manila/share/drivers/netapp/dataontap/client/client_base.py

@@ -32,7 +32,9 @@ class NetAppBaseClient(object):
             port=kwargs['port'],
             username=kwargs['username'],
             password=kwargs['password'],
-            trace=kwargs.get('trace', False))
+            trace=kwargs.get('trace', False),
+            api_trace_pattern=kwargs.get('api_trace_pattern',
+                                         na_utils.API_TRACE_PATTERN))
 
     def get_ontapi_version(self, cached=True):
         """Gets the supported ontapi version."""

manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py

@@ -117,7 +117,8 @@ class NetAppCmodeFileStorageLibrary(object):
 
         self._app_version = kwargs.get('app_version', 'unknown')
 
-        na_utils.setup_tracing(self.configuration.netapp_trace_flags)
+        na_utils.setup_tracing(self.configuration.netapp_trace_flags,
+                               self.configuration.netapp_api_trace_pattern)
         self._backend_name = self.configuration.safe_get(
             'share_backend_name') or driver_name
 
@@ -150,7 +151,8 @@ class NetAppCmodeFileStorageLibrary(object):
                 hostname=self.configuration.netapp_server_hostname,
                 port=self.configuration.netapp_server_port,
                 vserver=vserver,
-                trace=na_utils.TRACE_API)
+                trace=na_utils.TRACE_API,
+                api_trace_pattern=na_utils.API_TRACE_PATTERN)
             self._clients[vserver] = client
 
         return client

manila/share/drivers/netapp/options.py

@@ -115,7 +115,16 @@ netapp_support_opts = [
     cfg.StrOpt('netapp_trace_flags',
                help=('Comma-separated list of options that control which '
                      'trace info is written to the debug logs.  Values '
-                     'include method and api.')), ]
+                     'include method and api. API logging can further be '
+                     'filtered with the '
+                     '``netapp_api_trace_pattern option``.')),
+    cfg.StrOpt('netapp_api_trace_pattern',
+               default='(.*)',
+               help=('A regular expression to limit the API tracing. This '
+                     'option is honored only if enabling ``api`` tracing '
+                     'with the ``netapp_trace_flags`` option. By default, '
+                     'all APIs will be traced.')),
+]
 
 netapp_data_motion_opts = [
     cfg.IntOpt('netapp_snapmirror_quiesce_timeout',

manila/share/drivers/netapp/utils.py

@@ -18,6 +18,7 @@
 import collections
 import decimal
 import platform
+import re
 
 from oslo_concurrency import processutils as putils
 from oslo_log import log
@@ -33,6 +34,7 @@ LOG = log.getLogger(__name__)
 VALID_TRACE_FLAGS = ['method', 'api']
 TRACE_METHOD = False
 TRACE_API = False
+API_TRACE_PATTERN = '(.*)'
 
 
 def validate_driver_instantiation(**kwargs):
@@ -65,16 +67,24 @@ def round_down(value, precision='0.00'):
         decimal.Decimal(precision), rounding=decimal.ROUND_DOWN))
 
 
-def setup_tracing(trace_flags_string):
+def setup_tracing(trace_flags_string, api_trace_pattern=API_TRACE_PATTERN):
     global TRACE_METHOD
     global TRACE_API
+    global API_TRACE_PATTERN
     TRACE_METHOD = False
     TRACE_API = False
+    API_TRACE_PATTERN = api_trace_pattern
     if trace_flags_string:
         flags = trace_flags_string.split(',')
         flags = [flag.strip() for flag in flags]
         for invalid_flag in list(set(flags) - set(VALID_TRACE_FLAGS)):
             LOG.warning('Invalid trace flag: %s', invalid_flag)
+        try:
+            re.compile(api_trace_pattern)
+        except re.error:
+            msg = _('Cannot parse the API trace pattern. %s is not a '
+                    'valid python regular expression.') % api_trace_pattern
+            raise exception.BadConfigurationException(reason=msg)
         TRACE_METHOD = 'method' in flags
         TRACE_API = 'api' in flags
 

manila/tests/share/drivers/netapp/dataontap/client/fakes.py

@@ -24,7 +24,8 @@ CONNECTION_INFO = {
     'transport_type': 'https',
     'port': 443,
     'username': 'admin',
-    'password': 'passw0rd'
+    'password': 'passw0rd',
+    'api_trace_pattern': '(.*)',
 }
 
 CLUSTER_NAME = 'fake_cluster'
@@ -2395,7 +2396,7 @@ QOS_POLICY_GROUP_GET_ITER_RESPONSE = etree.XML("""
     'max_througput': QOS_MAX_THROUGHPUT,
 })
 
-FAKE_VOL_XML = """<volume-info xmlns='http://www.netapp.com/filer/admin'>
+FAKE_VOL_XML = """<volume-info>
     <name>open123</name>
     <state>online</state>
     <size-total>0</size-total>

manila/tests/share/drivers/netapp/dataontap/client/test_api.py

@@ -220,10 +220,20 @@ class NetAppApiServerTests(test.TestCase):
                                       na_element)
         self.assertEqual('unknown', exception.code)
 
-    def test_invoke_elem_valid(self):
+    @ddt.data({'trace_enabled': False,
+               'trace_pattern': '(.*)', 'log': False},
+              {'trace_enabled': True,
+               'trace_pattern': '(?!(volume)).*', 'log': False},
+              {'trace_enabled': True,
+               'trace_pattern': '(.*)', 'log': True},
+              {'trace_enabled': True,
+               'trace_pattern': '^volume-(info|get-iter)$', 'log': True})
+    @ddt.unpack
+    def test_invoke_elem_valid(self, trace_enabled, trace_pattern, log):
         """Tests the method invoke_elem with valid parameters"""
         na_element = fake.FAKE_NA_ELEMENT
-        self.root._trace = True
+        self.root._trace = trace_enabled
+        self.root._api_trace_pattern = trace_pattern
         self.mock_object(self.root, '_create_request', mock.Mock(
             return_value=('abc', fake.FAKE_NA_ELEMENT)))
         self.mock_object(api, 'LOG')
@@ -237,4 +247,5 @@ class NetAppApiServerTests(test.TestCase):
 
         self.root.invoke_elem(na_element)
 
-        self.assertEqual(2, api.LOG.debug.call_count)
+        expected_log_count = 2 if log else 0
+        self.assertEqual(expected_log_count, api.LOG.debug.call_count)

manila/tests/share/drivers/netapp/dataontap/fakes.py

@@ -83,7 +83,8 @@ CLIENT_KWARGS = {
     'vserver': None,
     'transport_type': 'https',
     'password': 'pass',
-    'port': '443'
+    'port': '443',
+    'api_trace_pattern': '(.*)',
 }
 
 SHARE = {

manila/tests/share/drivers/netapp/test_utils.py

@@ -18,6 +18,7 @@ Mock unit tests for the NetApp driver utility module
 
 import platform
 
+import ddt
 import mock
 from oslo_concurrency import processutils as putils
 from oslo_log import log
@@ -28,6 +29,7 @@ from manila import test
 from manila import version
 
 
+@ddt.ddt
 class NetAppDriverUtilsTestCase(test.TestCase):
 
     def setUp(self):
@@ -56,19 +58,22 @@ class NetAppDriverUtilsTestCase(test.TestCase):
         self.assertAlmostEqual(na_utils.round_down(-5.567, '0'), -5)
 
     def test_setup_tracing(self):
-        na_utils.setup_tracing(None)
+        na_utils.setup_tracing(None, api_trace_pattern='(.*)')
         self.assertFalse(na_utils.TRACE_API)
         self.assertFalse(na_utils.TRACE_METHOD)
+        self.assertEqual('(.*)', na_utils.API_TRACE_PATTERN)
         self.assertEqual(0, na_utils.LOG.warning.call_count)
 
         na_utils.setup_tracing('method')
         self.assertFalse(na_utils.TRACE_API)
         self.assertTrue(na_utils.TRACE_METHOD)
+        self.assertEqual('(.*)', na_utils.API_TRACE_PATTERN)
         self.assertEqual(0, na_utils.LOG.warning.call_count)
 
-        na_utils.setup_tracing('method,api')
+        na_utils.setup_tracing('method,api', api_trace_pattern='(^fancy-api$)')
         self.assertTrue(na_utils.TRACE_API)
         self.assertTrue(na_utils.TRACE_METHOD)
+        self.assertEqual('(^fancy-api$)', na_utils.API_TRACE_PATTERN)
         self.assertEqual(0, na_utils.LOG.warning.call_count)
 
     def test_setup_tracing_invalid_key(self):
@@ -78,6 +83,12 @@ class NetAppDriverUtilsTestCase(test.TestCase):
         self.assertTrue(na_utils.TRACE_METHOD)
         self.assertEqual(1, na_utils.LOG.warning.call_count)
 
+    @ddt.data('?!(bad', '(reg]+', 'eX?!)')
+    def test_setup_tracing_invalid_regex(self, regex):
+        self.assertRaises(exception.BadConfigurationException,
+                          na_utils.setup_tracing, 'method,api',
+                          api_trace_pattern=regex)
+
     @na_utils.trace
     def _trace_test_method(*args, **kwargs):
         return 'OK'

releasenotes/notes/netapp-support-filtering-api-tracing-02d1f4271f44d24c.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - The NetApp driver supports a new configuration option
+    ``netapp_api_trace_pattern`` to enable filtering backend API
+    interactions to log. This option must be specified in the backend
+    section when desired and it accepts a valid python regular expression.