Add share usage size tracking

We need to have a way to gather information about actual share storage
usages, so cloud operators could use this information for billing,
health checks and/or other purposes.

Change-Id: Iaca1bb541a34af862b938e17e4a56d53de7a9cc4
Implement-Blueprint: share-usage-size

manila/share/driver.py

@@ -2422,3 +2422,27 @@ class ShareDriver(object):
         :param share_server: None or Share server model
         """
         raise NotImplementedError()
+
+    def update_share_usage_size(self, context, shares):
+        """Invoked to get the usage size of given shares.
+
+        Driver can use this method to update the share usage size of
+        the shares. To do that, a dictionary of shares should be
+        returned.
+        :param shares: None or a list of all shares for updates.
+        :returns: An empty list or a list of dictionary of updates in the
+        following format. The value of "used_size" can be specified in GiB
+        units, as a floating point number::
+
+            [
+                {
+                    'id': '09960614-8574-4e03-89cf-7cf267b0bd08',
+                    'used_size': '200',
+                    'gathered_at': datetime.datetime(2017, 8, 10, 15, 14, 6),
+                },
+            ]
+
+        """
+        LOG.debug("This backend does not support gathering 'used_size' of "
+                  "shares created on it.")
+        return []

manila/share/drivers/lvm.py

@@ -25,6 +25,7 @@ import re
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import importutils
+from oslo_utils import timeutils
 import six
 
 from manila import exception
@@ -444,3 +445,30 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver):
         helper.update_access(self.share_server,
                              snapshot['name'], access_rules,
                              add_rules=add_rules, delete_rules=delete_rules)
+
+    def update_share_usage_size(self, context, shares):
+        updated_shares = []
+        out, err = self._execute(
+            'df', '-l', '--output=target,used',
+            '--block-size=g')
+        gathered_at = timeutils.utcnow()
+
+        for share in shares:
+            try:
+                mount_path = self._get_mount_path(share)
+                if os.path.exists(mount_path):
+                    used_size = (re.findall(
+                        mount_path + "\s*[0-9.]+G", out)[0].
+                        split(' ')[-1][:-1])
+                    updated_shares.append({'id': share['id'],
+                                           'used_size': used_size,
+                                           'gathered_at': gathered_at})
+                else:
+                    raise exception.NotFound(
+                        _("Share mount path %s could not be "
+                          "found.") % mount_path)
+            except Exception:
+                LOG.exception("Failed to gather 'used_size' for share %s.",
+                              share['id'])
+
+        return updated_shares

manila/share/manager.py

@@ -104,6 +104,21 @@ share_manager_opts = [
                     'the share manager will poll the driver to perform the '
                     'next step of migration in the storage backend, for a '
                     'migrating share.'),
+    cfg.IntOpt('share_usage_size_update_interval',
+               default=300,
+               help='This value, specified in seconds, determines how often '
+                    'the share manager will poll the driver to update the '
+                    'share usage size in the storage backend, for shares in '
+                    'that backend.'),
+    cfg.BoolOpt('enable_gathering_share_usage_size',
+                default=False,
+                help='If set to True, share usage size will be polled for in '
+                     'the interval specified with '
+                     '"share_usage_size_update_interval". Usage data can be '
+                     'consumed by telemetry integration. If telemetry is not '
+                     'configured, this option must be set to False. '
+                     'If set to False - gathering share usage size will be'
+                     ' disabled.'),
 ]
 
 CONF = cfg.CONF
@@ -3849,3 +3864,28 @@ class ShareManager(manager.SchedulerDependentManager):
         share_utils.notify_about_share_usage(
             context, share, share_instance, event_suffix,
             extra_usage_info=extra_usage_info, host=self.host)
+
+    @periodic_task.periodic_task(
+        spacing=CONF.share_usage_size_update_interval,
+        enabled=CONF.enable_gathering_share_usage_size)
+    @utils.require_driver_initialized
+    def update_share_usage_size(self, context):
+        """Invokes driver to gather usage size of shares."""
+        updated_share_instances = []
+        share_instances = self.db.share_instances_get_all_by_host(
+            context, host=self.host, with_share_data=True)
+
+        if share_instances:
+            try:
+                updated_share_instances = self.driver.update_share_usage_size(
+                    context, share_instances)
+            except Exception:
+                LOG.exception("Gather share usage size failure.")
+
+        for si in updated_share_instances:
+            share_instance = self._get_share_instance(context, si['id'])
+            share = self.db.share_get(context, share_instance['share_id'])
+            self._notify_about_share_usage(
+                context, share, share_instance, "consumed.size",
+                extra_usage_info={'used_size': si['used_size'],
+                                  'gathered_at': si['gathered_at']})

manila/tests/share/drivers/dummy.py

@@ -34,6 +34,7 @@ import time
 
 from oslo_config import cfg
 from oslo_log import log
+from oslo_utils import timeutils
 
 from manila.common import constants
 from manila import exception
@@ -628,3 +629,12 @@ class DummyDriver(driver.ShareDriver):
                       "progress": total_progress
                   })
         return {"total_progress": total_progress}
+
+    def update_share_usage_size(self, context, shares):
+        share_updates = []
+        gathered_at = timeutils.utcnow()
+        for s in shares:
+            share_updates.append({'id': s['id'],
+                                  'used_size':  1,
+                                  'gathered_at': gathered_at})
+        return share_updates

manila/tests/share/drivers/test_lvm.py

@@ -19,6 +19,7 @@ import os
 import ddt
 import mock
 from oslo_config import cfg
+from oslo_utils import timeutils
 
 from manila.common import constants as const
 from manila import context
@@ -591,3 +592,64 @@ class LVMShareDriverTestCase(test.TestCase):
             update_access.assert_called_once_with(
             self.server, self.snapshot['name'],
             access_rules, add_rules=add_rules, delete_rules=delete_rules))
+
+    @mock.patch.object(timeutils, 'utcnow', mock.Mock(
+                       return_value='fake_date'))
+    def test_update_share_usage_size(self):
+        mount_path = self._get_mount_path(self.share)
+        self._os.path.exists.return_value = True
+        self.mock_object(
+            self._driver,
+            '_execute',
+            mock.Mock(return_value=(
+                "Mounted on                    Used "
+                + mount_path + "               1G", None)))
+
+        update_shares = self._driver.update_share_usage_size(
+            self._context, [self.share, ])
+        self._os.path.exists.assert_called_with(mount_path)
+        self.assertEqual(
+            [{'id': 'fakeid', 'used_size': '1',
+              'gathered_at': 'fake_date'}],
+            update_shares)
+        self._driver._execute.assert_called_once_with(
+            'df', '-l', '--output=target,used',
+            '--block-size=g')
+
+    @mock.patch.object(timeutils, 'utcnow', mock.Mock(
+                       return_value='fake_date'))
+    def test_update_share_usage_size_multiple_share(self):
+        share1 = fake_share(id='fakeid_get_fail', name='get_fail')
+        share2 = fake_share(id='fakeid_success', name='get_success')
+        share3 = fake_share(id='fakeid_not_exist', name='get_not_exist')
+
+        mount_path2 = self._get_mount_path(share2)
+        mount_path3 = self._get_mount_path(share3)
+        self._os.path.exists.side_effect = [True, True, False]
+        self.mock_object(
+            self._driver,
+            '_execute',
+            mock.Mock(return_value=(
+                "Mounted on                    Used "
+                + mount_path2 + "               1G", None)))
+
+        update_shares = self._driver.update_share_usage_size(
+            self._context, [share1, share2, share3])
+        self._os.path.exists.assert_called_with(mount_path3)
+        self.assertEqual(
+            [{'gathered_at': 'fake_date',
+              'id': 'fakeid_success', 'used_size': '1'}],
+            update_shares)
+        self._driver._execute.assert_called_with(
+            'df', '-l', '--output=target,used',
+            '--block-size=g')
+
+    def test_update_share_usage_size_fail(self):
+        def _fake_exec(*args, **kwargs):
+            raise exception.ProcessExecutionError(stderr="error")
+
+        self.mock_object(self._driver, '_execute', _fake_exec)
+        self.assertRaises(exception.ProcessExecutionError,
+                          self._driver.update_share_usage_size,
+                          self._context,
+                          [self.share])

manila/tests/share/test_manager.py

@@ -6303,6 +6303,46 @@ class ShareManagerTestCase(test.TestCase):
             self.context, share_instance['id'],
             share_server='fake_share_server')
 
+    @mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
+    def test_update_share_usage_size(self, mock_notify):
+        instances = self._setup_init_mocks(setup_access_rules=False)
+        update_shares = [{'id': 'fake_id', 'used_size': '3',
+                          'gathered_at': 'fake'}]
+        mock_notify.assert_not_called()
+
+        manager = self.share_manager
+        self.mock_object(manager, 'driver')
+        self.mock_object(manager.db, 'share_instances_get_all_by_host',
+                         mock.Mock(return_value=instances))
+        self.mock_object(manager.db, 'share_instance_get',
+                         mock.Mock(side_effect=instances))
+        mock_driver_call = self.mock_object(
+            manager.driver, 'update_share_usage_size',
+            mock.Mock(return_value=update_shares))
+        self.share_manager.update_share_usage_size(self.context)
+        self.assert_notify_called(mock_notify,
+                                  (['INFO', 'share.consumed.size'], ))
+        mock_driver_call.assert_called_once_with(
+            self.context, instances)
+
+    @mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
+    def test_update_share_usage_size_fail(self, mock_notify):
+        instances = self._setup_init_mocks(setup_access_rules=False)
+        mock_notify.assert_not_called()
+
+        self.mock_object(self.share_manager, 'driver')
+        self.mock_object(self.share_manager.db,
+                         'share_instances_get_all_by_host',
+                         mock.Mock(return_value=instances))
+        self.mock_object(self.share_manager.db, 'share_instance_get',
+                         mock.Mock(side_effect=instances))
+        self.mock_object(
+            self.share_manager.driver, 'update_share_usage_size',
+            mock.Mock(side_effect=exception.ProcessExecutionError))
+        mock_log_exception = self.mock_object(manager.LOG, 'exception')
+        self.share_manager.update_share_usage_size(self.context)
+        self.assertTrue(mock_log_exception.called)
+
 
 @ddt.ddt
 class HookWrapperTestCase(test.TestCase):

releasenotes/notes/add-gathering-usage-size-8454sd45deopb14e.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - Added periodic task to gather share usage size.
+upgrade:
+  - Added enable_gathering_share_usage_size and
+    share_usage_size_update_interval options in the
+    manila.conf file to allow configuration of gathering
+    share usage size support and to allow configuration
+    of interval time of gathering share usage size.