diff --git a/cinder/api/contrib/quotas.py b/cinder/api/contrib/quotas.py index b5e0bb954f8..574983e86b1 100644 --- a/cinder/api/contrib/quotas.py +++ b/cinder/api/contrib/quotas.py @@ -15,11 +15,6 @@ import webob -from keystoneclient.auth.identity.generic import token -from keystoneclient import client -from keystoneclient import exceptions -from keystoneclient import session - from cinder.api import extensions from cinder.api.openstack import wsgi from cinder.api import xmlutil @@ -28,6 +23,7 @@ from cinder.db.sqlalchemy import api as sqlalchemy_api from cinder import exception from cinder.i18n import _ from cinder import quota +from cinder import quota_utils from cinder import utils from oslo_config import cfg @@ -57,17 +53,6 @@ class QuotaTemplate(xmlutil.TemplateBuilder): class QuotaSetsController(wsgi.Controller): - class GenericProjectInfo(object): - - """Abstraction layer for Keystone V2 and V3 project objects""" - - def __init__(self, project_id, project_keystone_api_version, - project_parent_id=None, project_subtree=None): - self.id = project_id - self.keystone_api_version = project_keystone_api_version - self.parent_id = project_parent_id - self.subtree = project_subtree - def _format_quota_set(self, project_id, quota_set): """Convert the quota object to a result dict.""" @@ -75,20 +60,6 @@ class QuotaSetsController(wsgi.Controller): return dict(quota_set=quota_set) - def _keystone_client(self, context): - """Creates and returns an instance of a generic keystone client. - - :param context: The request context - :return: keystoneclient.client.Client object - """ - auth_plugin = token.Token( - auth_url=CONF.keystone_authtoken.auth_uri, - token=context.auth_token, - project_id=context.project_id) - client_session = session.Session(auth=auth_plugin) - return client.Client(auth_url=CONF.keystone_authtoken.auth_uri, - session=client_session) - def _validate_existing_resource(self, key, value, quota_values): if key == 'per_volume_gigabytes': return @@ -103,7 +74,11 @@ class QuotaSetsController(wsgi.Controller): limit = self.validate_integer(quota[key], key, min_value=-1, max_value=db.MAX_INT) - if parent_project_quotas: + # If a parent quota is unlimited (-1) no validation needs to happen + # for the amount of existing free quota + # TODO(mc_nair): will need to recurse up for nested quotas once + # -1 child project values are enabled + if parent_project_quotas and parent_project_quotas[key]['limit'] != -1: free_quota = (parent_project_quotas[key]['limit'] - parent_project_quotas[key]['in_use'] - parent_project_quotas[key]['reserved'] - @@ -112,15 +87,22 @@ class QuotaSetsController(wsgi.Controller): current = 0 if project_quotas.get(key): current = project_quotas[key]['limit'] + # -1 limit doesn't change free quota available in parent + if current == -1: + current = 0 - if limit - current > free_quota: + # Add back the existing quota limit (if any is set) from the + # current free quota since it will be getting reset and is part + # of the parent's allocated value + free_quota += current + + if limit > free_quota: msg = _("Free quota available is %s.") % free_quota raise webob.exc.HTTPBadRequest(explanation=msg) return limit - def _get_quotas(self, context, id, usages=False, parent_project_id=None): - values = QUOTAS.get_project_quotas(context, id, usages=usages, - parent_project_id=parent_project_id) + def _get_quotas(self, context, id, usages=False): + values = QUOTAS.get_project_quotas(context, id, usages=usages) if usages: return values @@ -199,27 +181,6 @@ class QuotaSetsController(wsgi.Controller): return True return False - def _get_project(self, context, id, subtree_as_ids=False): - """A Helper method to get the project hierarchy. - - Along with Hierachical Multitenancy in keystone API v3, projects can be - hierarchically organized. Therefore, we need to know the project - hierarchy, if any, in order to do quota operations properly. - """ - try: - keystone = self._keystone_client(context) - generic_project = self.GenericProjectInfo(id, keystone.version) - if keystone.version == 'v3': - project = keystone.projects.get(id, - subtree_as_ids=subtree_as_ids) - generic_project.parent_id = project.parent_id - generic_project.subtree = ( - project.subtree if subtree_as_ids else None) - except exceptions.NotFound: - msg = (_("Tenant ID: %s does not exist.") % id) - raise webob.exc.HTTPNotFound(explanation=msg) - return generic_project - @wsgi.serializers(xml=QuotaTemplate) def show(self, req, id): """Show quota for a particular tenant @@ -242,21 +203,16 @@ class QuotaSetsController(wsgi.Controller): else: usage = False - try: + if QUOTAS.using_nested_quotas(): # With hierarchical projects, only the admin of the current project # or the root project has privilege to perform quota show # operations. - target_project = self._get_project(context, target_project_id) - context_project = self._get_project(context, context.project_id, - subtree_as_ids=True) + target_project = quota_utils.get_project_hierarchy( + context, target_project_id) + context_project = quota_utils.get_project_hierarchy( + context, context.project_id, subtree_as_ids=True) self._authorize_show(context_project, target_project) - parent_project_id = target_project.parent_id - except exceptions.Forbidden: - # NOTE(e0ne): Keystone API v2 requires admin permissions for - # project_get method. We ignore Forbidden exception for - # non-admin users. - parent_project_id = None try: sqlalchemy_api.authorize_project_context(context, @@ -264,8 +220,7 @@ class QuotaSetsController(wsgi.Controller): except exception.NotAuthorized: raise webob.exc.HTTPForbidden() - quotas = self._get_quotas(context, target_project_id, usage, - parent_project_id=parent_project_id) + quotas = self._get_quotas(context, target_project_id, usage) return self._format_quota_set(target_project_id, quotas) @wsgi.serializers(xml=QuotaTemplate) @@ -311,22 +266,25 @@ class QuotaSetsController(wsgi.Controller): msg = _("Bad key(s) in quota set: %s") % ",".join(bad_keys) raise webob.exc.HTTPBadRequest(explanation=msg) - # Get the parent_id of the target project to verify whether we are - # dealing with hierarchical namespace or non-hierarchical namespace. - target_project = self._get_project(context, target_project_id) - parent_id = target_project.parent_id + # Saving off this value since we need to use it multiple times + use_nested_quotas = QUOTAS.using_nested_quotas() + if use_nested_quotas: + # Get the parent_id of the target project to verify whether we are + # dealing with hierarchical namespace or non-hierarchical namespace + target_project = quota_utils.get_project_hierarchy( + context, target_project_id) + parent_id = target_project.parent_id - if parent_id: - # Get the children of the project which the token is scoped to - # in order to know if the target_project is in its hierarchy. - context_project = self._get_project(context, - context.project_id, - subtree_as_ids=True) - self._authorize_update_or_delete(context_project, - target_project.id, - parent_id) - parent_project_quotas = QUOTAS.get_project_quotas( - context, parent_id) + if parent_id: + # Get the children of the project which the token is scoped to + # in order to know if the target_project is in its hierarchy. + context_project = quota_utils.get_project_hierarchy( + context, context.project_id, subtree_as_ids=True) + self._authorize_update_or_delete(context_project, + target_project.id, + parent_id) + parent_project_quotas = QUOTAS.get_project_quotas( + context, parent_id) # NOTE(ankit): Pass #2 - In this loop for body['quota_set'].keys(), # we validate the quota limits to ensure that we can bail out if @@ -344,10 +302,17 @@ class QuotaSetsController(wsgi.Controller): if not skip_flag: self._validate_existing_resource(key, value, quota_values) - if parent_id: + if use_nested_quotas and parent_id: value = self._validate_quota_limit(body['quota_set'], key, quota_values, parent_project_quotas) + + if value < 0: + # TODO(mc_nair): extend to handle -1 limits and recurse up + # the hierarchy + msg = _("Quota can't be set to -1 for child projects.") + raise webob.exc.HTTPBadRequest(explanation=msg) + original_quota = 0 if quota_values.get(key): original_quota = quota_values[key]['limit'] @@ -373,7 +338,7 @@ class QuotaSetsController(wsgi.Controller): # If hierarchical projects, update child's quota first # and then parents quota. In future this needs to be an # atomic operation. - if parent_id: + if use_nested_quotas and parent_id: if key in allocated_quotas.keys(): try: db.quota_allocated_update(context, parent_id, key, @@ -383,24 +348,15 @@ class QuotaSetsController(wsgi.Controller): db.quota_create(context, parent_id, key, parent_limit, allocated=allocated_quotas[key]) - return {'quota_set': self._get_quotas(context, target_project_id, - parent_project_id=parent_id)} + return {'quota_set': self._get_quotas(context, target_project_id)} @wsgi.serializers(xml=QuotaTemplate) def defaults(self, req, id): context = req.environ['cinder.context'] authorize_show(context) - try: - project = self._get_project(context, context.project_id) - parent_id = project.parent_id - except exceptions.Forbidden: - # NOTE(e0ne): Keystone API v2 requires admin permissions for - # project_get method. We ignore Forbidden exception for - # non-admin users. - parent_id = context.project_id return self._format_quota_set(id, QUOTAS.get_defaults( - context, parent_project_id=parent_id)) + context, project_id=id)) @wsgi.serializers(xml=QuotaTemplate) def delete(self, req, id): @@ -416,20 +372,30 @@ class QuotaSetsController(wsgi.Controller): context = req.environ['cinder.context'] authorize_delete(context) - # Get the parent_id of the target project to verify whether we are - # dealing with hierarchical namespace or non-hierarchical namespace. - target_project = self._get_project(context, id) - parent_id = target_project.parent_id + if QUOTAS.using_nested_quotas(): + self._delete_nested_quota(context, id) + else: + try: + db.quota_destroy_by_project(context, id) + except exception.AdminRequired: + raise webob.exc.HTTPForbidden() + def _delete_nested_quota(self, ctxt, proj_id): + # Get the parent_id of the target project to verify whether we are + # dealing with hierarchical namespace or non-hierarchical + # namespace. try: project_quotas = QUOTAS.get_project_quotas( - context, target_project.id, usages=True, - parent_project_id=parent_id, defaults=False) + ctxt, proj_id, usages=True, defaults=False) except exception.NotAuthorized: raise webob.exc.HTTPForbidden() - # If the project which is being deleted has allocated part of its quota - # to its subprojects, then subprojects' quotas should be deleted first. + target_project = quota_utils.get_project_hierarchy( + ctxt, proj_id) + parent_id = target_project.parent_id + # If the project which is being deleted has allocated part of its + # quota to its subprojects, then subprojects' quotas should be + # deleted first. for key, value in project_quotas.items(): if 'allocated' in project_quotas[key].keys(): if project_quotas[key]['allocated'] != 0: @@ -438,35 +404,48 @@ class QuotaSetsController(wsgi.Controller): raise webob.exc.HTTPBadRequest(explanation=msg) if parent_id: - # Get the children of the project which the token is scoped to in - # order to know if the target_project is in its hierarchy. - context_project = self._get_project(context, - context.project_id, - subtree_as_ids=True) + # Get the children of the project which the token is scoped to + # in order to know if the target_project is in its hierarchy. + context_project = quota_utils.get_project_hierarchy( + ctxt, ctxt.project_id, subtree_as_ids=True) self._authorize_update_or_delete(context_project, target_project.id, parent_id) parent_project_quotas = QUOTAS.get_project_quotas( - context, parent_id, parent_project_id=parent_id) + ctxt, parent_id) # Delete child quota first and later update parent's quota. try: - db.quota_destroy_by_project(context, target_project.id) + db.quota_destroy_by_project(ctxt, target_project.id) except exception.AdminRequired: raise webob.exc.HTTPForbidden() - # Update the allocated of the parent + # The parent "gives" quota to its child using the "allocated" value + # and since the child project is getting deleted, we should restore + # the child projects quota to the parent quota, but lowering it's + # allocated value for key, value in project_quotas.items(): project_hard_limit = project_quotas[key]['limit'] parent_allocated = parent_project_quotas[key]['allocated'] parent_allocated -= project_hard_limit - db.quota_allocated_update(context, parent_id, key, + db.quota_allocated_update(ctxt, parent_id, key, parent_allocated) - else: - try: - db.quota_destroy_by_project(context, target_project.id) - except exception.AdminRequired: - raise webob.exc.HTTPForbidden() + + def validate_setup_for_nested_quota_use(self, req): + """Validates that the setup supports using nested quotas. + + Ensures that Keystone v3 or greater is being used, and that the + existing quotas make sense to nest in the current hierarchy (e.g. that + no child quota would be larger than it's parent). + """ + ctxt = req.environ['cinder.context'] + params = req.params + try: + quota_utils.validate_setup_for_nested_quota_use( + ctxt, QUOTAS.resources, quota.NestedDbQuotaDriver(), + fix_allocated_quotas=params.get('fix_allocated_quotas')) + except exception.InvalidNestedQuotaSetup as e: + raise webob.exc.HTTPBadRequest(explanation=e.msg) class Quotas(extensions.ExtensionDescriptor): @@ -480,9 +459,10 @@ class Quotas(extensions.ExtensionDescriptor): def get_resources(self): resources = [] - res = extensions.ResourceExtension('os-quota-sets', - QuotaSetsController(), - member_actions={'defaults': 'GET'}) + res = extensions.ResourceExtension( + 'os-quota-sets', QuotaSetsController(), + member_actions={'defaults': 'GET'}, + collection_actions={'validate_setup_for_nested_quota_use': 'GET'}) resources.append(res) return resources diff --git a/cinder/exception.py b/cinder/exception.py index 1911a57e828..ce5cfe57d15 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -376,6 +376,11 @@ class InvalidQuotaValue(Invalid): "resources: %(unders)s") +class InvalidNestedQuotaSetup(CinderException): + message = _("Project quotas are not properly setup for nested quotas: " + "%(reason)s.") + + class QuotaNotFound(NotFound): message = _("Quota could not be found") diff --git a/cinder/quota.py b/cinder/quota.py index f32931db2ed..bf5974049c4 100644 --- a/cinder/quota.py +++ b/cinder/quota.py @@ -16,7 +16,7 @@ """Quotas for volumes.""" - +from collections import deque import datetime from oslo_config import cfg @@ -30,6 +30,7 @@ from cinder import context from cinder import db from cinder import exception from cinder.i18n import _, _LE +from cinder import quota_utils LOG = logging.getLogger(__name__) @@ -65,7 +66,7 @@ quota_opts = [ default=0, help='Number of seconds between subsequent usage refreshes'), cfg.StrOpt('quota_driver', - default='cinder.quota.DbQuotaDriver', + default="cinder.quota.DbQuotaDriver", help='Default driver to use for quota checks'), cfg.BoolOpt('use_default_quota_class', default=True, @@ -97,18 +98,12 @@ class DbQuotaDriver(object): return db.quota_class_get(context, quota_class, resource_name) - def get_default(self, context, resource, parent_project_id=None): - """Get a specific default quota for a resource. - - :param parent_project_id: The id of the current project's parent, - if any. - """ - + def get_default(self, context, resource, project_id): + """Get a specific default quota for a resource.""" default_quotas = db.quota_class_get_default(context) - default_quota_value = 0 if parent_project_id else resource.default - return default_quotas.get(resource.name, default_quota_value) + return default_quotas.get(resource.name, resource.default) - def get_defaults(self, context, resources, parent_project_id=None): + def get_defaults(self, context, resources, project_id=None): """Given a list of resources, retrieve the default quotas. Use the class quotas named `_DEFAULT_QUOTA_NAME` as default quotas, @@ -116,13 +111,12 @@ class DbQuotaDriver(object): :param context: The request context, for access checks. :param resources: A dictionary of the registered resources. - :param parent_project_id: The id of the current project's parent, - if any. + :param project_id: The id of the current project """ quotas = {} default_quotas = {} - if CONF.use_default_quota_class and not parent_project_id: + if CONF.use_default_quota_class: default_quotas = db.quota_class_get_default(context) for resource in resources.values(): @@ -135,8 +129,7 @@ class DbQuotaDriver(object): "default quota class for default " "quota.") % {'res': resource.name}) quotas[resource.name] = default_quotas.get(resource.name, - (0 if parent_project_id - else resource.default)) + resource.default) return quotas def get_class_quotas(self, context, resources, quota_class, @@ -170,7 +163,7 @@ class DbQuotaDriver(object): def get_project_quotas(self, context, resources, project_id, quota_class=None, defaults=True, - usages=True, parent_project_id=None): + usages=True): """Retrieve quotas for a project. Given a list of resources, retrieve the quotas for the given @@ -190,12 +183,11 @@ class DbQuotaDriver(object): specific value for the resource. :param usages: If True, the current in_use, reserved and allocated counts will also be returned. - :param parent_project_id: The id of the current project's parent, - if any. """ quotas = {} project_quotas = db.quota_get_all_by_project(context, project_id) + allocated_quotas = None if usages: project_usages = db.quota_usage_get_all_by_project(context, project_id) @@ -214,8 +206,8 @@ class DbQuotaDriver(object): else: class_quotas = {} - default_quotas = self.get_defaults(context, resources, - parent_project_id=parent_project_id) + # TODO(mc_nair): change this to be lazy loaded + default_quotas = self.get_defaults(context, resources, project_id) for resource in resources.values(): # Omit default/quota class values @@ -237,15 +229,12 @@ class DbQuotaDriver(object): quotas[resource.name].update( in_use=usage.get('in_use', 0), reserved=usage.get('reserved', 0), ) - - if parent_project_id or allocated_quotas: - quotas[resource.name].update( - allocated=allocated_quotas.get(resource.name, 0), ) - + if allocated_quotas: + quotas[resource.name].update( + allocated=allocated_quotas.get(resource.name, 0), ) return quotas - def _get_quotas(self, context, resources, keys, has_sync, project_id=None, - parent_project_id=None): + def _get_quotas(self, context, resources, keys, has_sync, project_id=None): """A helper method which retrieves the quotas for specific resources. This specific resource is identified by keys, and which apply to the @@ -261,8 +250,6 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. - :param parent_project_id: The id of the current project's parent, - if any. """ # Filter resources @@ -282,8 +269,7 @@ class DbQuotaDriver(object): # Grab and return the quotas (without usages) quotas = self.get_project_quotas(context, sub_resources, project_id, - context.quota_class, usages=False, - parent_project_id=parent_project_id) + context.quota_class, usages=False) return {k: v['limit'] for k, v in quotas.items()} @@ -452,6 +438,134 @@ class DbQuotaDriver(object): db.reservation_expire(context) +class NestedDbQuotaDriver(DbQuotaDriver): + def validate_nested_setup(self, ctxt, resources, project_tree, + fix_allocated_quotas=False): + """Ensures project_tree has quotas that make sense as nested quotas. + + Validates the following: + * No child projects have a limit of -1 + * No parent project has child_projects who have more combined quota + than the parent's quota limit + * No child quota has a larger in-use value than it's current limit + (could happen before because child default values weren't enforced) + * All parent projects' "allocated" quotas match the sum of the limits + of its children projects + """ + project_queue = deque(project_tree.items()) + borked_allocated_quotas = {} + + while project_queue: + # Tuple of (current root node, subtree) + cur_project_id, project_subtree = project_queue.popleft() + + # If we're on a leaf node, no need to do validation on it, and in + # order to avoid complication trying to get its children, skip it. + if not project_subtree: + continue + + cur_project_quotas = self.get_project_quotas( + ctxt, resources, cur_project_id) + + child_project_ids = project_subtree.keys() + child_project_quotas = {child_id: self.get_project_quotas( + ctxt, resources, child_id) for child_id in child_project_ids} + + # Validate each resource when compared to it's child quotas + for resource in cur_project_quotas.keys(): + child_limit_sum = 0 + for child_id, child_quota in child_project_quotas.items(): + child_limit = child_quota[resource]['limit'] + # Don't want to continue validation if -1 limit for child + # TODO(mc_nair) - remove when allowing -1 for subprojects + if child_limit < 0: + msg = _("Quota limit is -1 for child project " + "'%(proj)s' for resource '%(res)s'") % { + 'proj': child_id, 'res': resource + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + # Handle the case that child default quotas weren't being + # properly enforced before + elif child_quota[resource].get('in_use', 0) > child_limit: + msg = _("Quota limit invalid for project '%(proj)s' " + "for resource '%(res)s': limit of %(limit)d " + "is less than in-use value of %(used)d") % { + 'proj': child_id, 'res': resource, + 'limit': child_limit, + 'used': child_quota[resource]['in_use'] + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + + child_limit_sum += child_quota[resource]['limit'] + + parent_quota = cur_project_quotas[resource] + parent_limit = parent_quota['limit'] + parent_usage = parent_quota['in_use'] + parent_allocated = parent_quota.get('allocated', 0) + + if parent_limit > 0: + parent_free_quota = parent_limit - parent_usage + if parent_free_quota < child_limit_sum: + msg = _("Sum of child limits '%(sum)s' is greater " + "than free quota of '%(free)s' for project " + "'%(proj)s' for resource '%(res)s'. Please " + "lower the limit for one or more of the " + "following projects: '%(child_ids)s'") % { + 'sum': child_limit_sum, 'free': parent_free_quota, + 'proj': cur_project_id, 'res': resource, + 'child_ids': ', '.join(child_project_ids) + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + + # Deal with the fact that using -1 limits in the past may + # have messed some allocated values in DB + if parent_allocated != child_limit_sum: + # Decide whether to fix the allocated val or just + # keep track of what's messed up + if fix_allocated_quotas: + try: + db.quota_allocated_update(ctxt, cur_project_id, + resource, + child_limit_sum) + except exception.ProjectQuotaNotFound: + # Handles the case that the project is using + # default quota value so nothing present to update + db.quota_create( + ctxt, cur_project_id, resource, + parent_limit, allocated=child_limit_sum) + else: + if cur_project_id not in borked_allocated_quotas: + borked_allocated_quotas[cur_project_id] = {} + + borked_allocated_quotas[cur_project_id][resource] = { + 'db_allocated_quota': parent_allocated, + 'expected_allocated_quota': child_limit_sum} + + project_queue.extend(project_subtree.items()) + + if borked_allocated_quotas: + msg = _("Invalid allocated quotas defined for the following " + "project quotas: %s") % borked_allocated_quotas + raise exception.InvalidNestedQuotaSetup(message=msg) + + def get_default(self, context, resource, project_id): + """Get a specific default quota for a resource.""" + resource = super(NestedDbQuotaDriver, self).get_default( + context, resource, project_id) + + return 0 if quota_utils.get_parent_project_id( + context, project_id) else resource.default + + def get_defaults(self, context, resources, project_id=None): + defaults = super(NestedDbQuotaDriver, self).get_defaults( + context, resources, project_id) + # All defaults are 0 for child project + if quota_utils.get_parent_project_id(context, project_id): + for key in defaults.keys(): + defaults[key] = 0 + return defaults + + class BaseResource(object): """Describe a single resource for quota checking.""" @@ -626,14 +740,31 @@ class QuotaEngine(object): def __init__(self, quota_driver_class=None): """Initialize a Quota object.""" - if not quota_driver_class: - quota_driver_class = CONF.quota_driver - - if isinstance(quota_driver_class, six.string_types): - quota_driver_class = importutils.import_object(quota_driver_class) - self._resources = {} - self._driver = quota_driver_class + self._quota_driver_class = quota_driver_class + self._driver_class = None + + @property + def _driver(self): + # Lazy load the driver so we give a chance for the config file to + # be read before grabbing the config for which QuotaDriver to use + if self._driver_class: + return self._driver_class + + if not self._quota_driver_class: + # Grab the current driver class from CONF + self._quota_driver_class = CONF.quota_driver + + if isinstance(self._quota_driver_class, six.string_types): + self._quota_driver_class = importutils.import_object( + self._quota_driver_class) + + self._driver_class = self._quota_driver_class + return self._driver_class + + def using_nested_quotas(self): + """Returns true if nested quotas are being used""" + return isinstance(self._driver, NestedDbQuotaDriver) def __contains__(self, resource): return resource in self.resources @@ -669,16 +800,15 @@ class QuotaEngine(object): return self._driver.get_default(context, resource, parent_project_id=parent_project_id) - def get_defaults(self, context, parent_project_id=None): + def get_defaults(self, context, project_id=None): """Retrieve the default quotas. :param context: The request context, for access checks. - :param parent_project_id: The id of the current project's parent, - if any. + :param project_id: The id of the current project """ return self._driver.get_defaults(context, self.resources, - parent_project_id) + project_id) def get_class_quotas(self, context, quota_class, defaults=True): """Retrieve the quotas for the given quota class. @@ -695,7 +825,7 @@ class QuotaEngine(object): quota_class, defaults=defaults) def get_project_quotas(self, context, project_id, quota_class=None, - defaults=True, usages=True, parent_project_id=None): + defaults=True, usages=True): """Retrieve the quotas for the given project. :param context: The request context, for access checks. @@ -709,17 +839,12 @@ class QuotaEngine(object): specific value for the resource. :param usages: If True, the current in_use, reserved and allocated counts will also be returned. - :param parent_project_id: The id of the current project's parent, - if any. """ - return self._driver.get_project_quotas(context, self.resources, project_id, quota_class=quota_class, defaults=defaults, - usages=usages, - parent_project_id= - parent_project_id) + usages=usages) def count(self, context, resource, *args, **kwargs): """Count a resource. diff --git a/cinder/quota_utils.py b/cinder/quota_utils.py index ca5c0817516..89de099920d 100644 --- a/cinder/quota_utils.py +++ b/cinder/quota_utils.py @@ -12,20 +12,42 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import webob +from oslo_config import cfg from oslo_log import log as logging +from keystoneclient.auth.identity.generic import token +from keystoneclient import client +from keystoneclient import exceptions +from keystoneclient import session + from cinder import exception -from cinder.i18n import _LW -from cinder import quota +from cinder.i18n import _, _LW + +CONF = cfg.CONF LOG = logging.getLogger(__name__) -QUOTAS = quota.QUOTAS + + +class GenericProjectInfo(object): + """Abstraction layer for Keystone V2 and V3 project objects""" + def __init__(self, project_id, project_keystone_api_version, + project_parent_id=None, + project_subtree=None, + project_parent_tree=None): + self.id = project_id + self.keystone_api_version = project_keystone_api_version + self.parent_id = project_parent_id + self.subtree = project_subtree + self.parents = project_parent_tree def get_volume_type_reservation(ctxt, volume, type_id, reserve_vol_type_only=False): + from cinder import quota + QUOTAS = quota.QUOTAS # Reserve quotas for the given volume type try: reserve_opts = {'volumes': 1, 'gigabytes': volume['size']} @@ -78,3 +100,104 @@ def get_volume_type_reservation(ctxt, volume, type_id, raise exception.VolumeLimitExceeded( allowed=quotas[over]) return reservations + + +def get_project_hierarchy(context, project_id, subtree_as_ids=False): + """A Helper method to get the project hierarchy. + + Along with hierarchical multitenancy in keystone API v3, projects can be + hierarchically organized. Therefore, we need to know the project + hierarchy, if any, in order to do nested quota operations properly. + """ + try: + keystone = _keystone_client(context) + generic_project = GenericProjectInfo(project_id, keystone.version) + if keystone.version == 'v3': + project = keystone.projects.get(project_id, + subtree_as_ids=subtree_as_ids) + generic_project.parent_id = project.parent_id + generic_project.subtree = ( + project.subtree if subtree_as_ids else None) + except exceptions.NotFound: + msg = (_("Tenant ID: %s does not exist.") % project_id) + raise webob.exc.HTTPNotFound(explanation=msg) + + return generic_project + + +def get_parent_project_id(context, project_id): + return get_project_hierarchy(context, project_id).parent_id + + +def get_all_projects(context): + # Right now this would have to be done as cloud admin with Keystone v3 + return _keystone_client(context, (3, 0)).projects.list() + + +def get_all_root_project_ids(context): + project_list = get_all_projects(context) + + # Find every project which does not have a parent, meaning it is the + # root of the tree + project_roots = [project.id for project in project_list + if not project.parent_id] + + return project_roots + + +def validate_setup_for_nested_quota_use(ctxt, resources, + nested_quota_driver, + fix_allocated_quotas=False): + """Validates the setup supports using nested quotas. + + Ensures that Keystone v3 or greater is being used, that the current + user is of the cloud admin role, and that the existing quotas make sense to + nest in the current hierarchy (e.g. that no child quota would be larger + than it's parent). + + :param resources: the quota resources to validate + :param nested_quota_driver: nested quota driver used to validate each tree + :param fix_allocated_quotas: if True, parent projects "allocated" total + will be calculated based on the existing child limits and the DB will + be updated. If False, an exception is raised reporting any parent + allocated quotas are currently incorrect. + """ + try: + project_roots = get_all_root_project_ids(ctxt) + + # Now that we've got the roots of each tree, validate the trees + # to ensure that each is setup logically for nested quotas + for root in project_roots: + root_proj = get_project_hierarchy(ctxt, root, + subtree_as_ids=True) + nested_quota_driver.validate_nested_setup( + ctxt, + resources, + {root_proj.id: root_proj.subtree}, + fix_allocated_quotas=fix_allocated_quotas + ) + except exceptions.VersionNotAvailable: + msg = _("Keystone version 3 or greater must be used to get nested " + "quota support.") + raise exception.CinderException(message=msg) + except exceptions.Forbidden: + msg = _("Must run this command as cloud admin using " + "a Keystone policy.json which allows cloud " + "admin to list and get any project.") + raise exception.CinderException(message=msg) + + +def _keystone_client(context, version=(3, 0)): + """Creates and returns an instance of a generic keystone client. + + :param context: The request context + :param version: version of Keystone to request + :return: keystoneclient.client.Client object + """ + auth_plugin = token.Token( + auth_url=CONF.keystone_authtoken.auth_uri, + token=context.auth_token, + project_id=context.project_id) + client_session = session.Session(auth=auth_plugin) + return client.Client(auth_url=CONF.keystone_authtoken.auth_uri, + session=client_session, version=version) diff --git a/cinder/tests/unit/api/contrib/test_quotas.py b/cinder/tests/unit/api/contrib/test_quotas.py index 4bcc43fba9d..91e7fa18f46 100644 --- a/cinder/tests/unit/api/contrib/test_quotas.py +++ b/cinder/tests/unit/api/contrib/test_quotas.py @@ -29,10 +29,10 @@ import webob.exc from cinder.api.contrib import quotas from cinder import context from cinder import db +from cinder import quota from cinder import test from cinder.tests.unit import test_db_api -from keystoneclient import exceptions from keystonemiddleware import auth_token from oslo_config import cfg from oslo_config import fixture as config_fixture @@ -43,7 +43,7 @@ CONF = cfg.CONF def make_body(root=True, gigabytes=1000, snapshots=10, volumes=10, backups=10, backup_gigabytes=1000, - tenant_id='foo', per_volume_gigabytes=-1): + tenant_id='foo', per_volume_gigabytes=-1, is_child=False): resources = {'gigabytes': gigabytes, 'snapshots': snapshots, 'volumes': volumes, @@ -52,10 +52,18 @@ def make_body(root=True, gigabytes=1000, snapshots=10, 'per_volume_gigabytes': per_volume_gigabytes, } # need to consider preexisting volume types as well volume_types = db.volume_type_get_all(context.get_admin_context()) - for volume_type in volume_types: - resources['gigabytes_' + volume_type] = -1 - resources['snapshots_' + volume_type] = -1 - resources['volumes_' + volume_type] = -1 + + if not is_child: + for volume_type in volume_types: + resources['gigabytes_' + volume_type] = -1 + resources['snapshots_' + volume_type] = -1 + resources['volumes_' + volume_type] = -1 + elif per_volume_gigabytes < 0: + # In the case that we're dealing with a child project, we aren't + # allowing -1 limits for the time being, so hack this to some large + # enough value for the tests that it's essentially unlimited + # TODO(mc_nair): remove when -1 limits for child projects are allowed + resources['per_volume_gigabytes'] = 10000 if tenant_id: resources['id'] = tenant_id @@ -75,7 +83,7 @@ def make_subproject_body(root=True, gigabytes=0, snapshots=0, per_volume_gigabytes=per_volume_gigabytes) -class QuotaSetsControllerTest(test.TestCase): +class QuotaSetsControllerTestBase(test.TestCase): class FakeProject(object): @@ -85,15 +93,30 @@ class QuotaSetsControllerTest(test.TestCase): self.subtree = None def setUp(self): - super(QuotaSetsControllerTest, self).setUp() + super(QuotaSetsControllerTestBase, self).setUp() + self.controller = quotas.QuotaSetsController() self.req = mock.Mock() self.req.environ = {'cinder.context': context.get_admin_context()} self.req.environ['cinder.context'].is_admin = True + self.req.params = {} self._create_project_hierarchy() + get_patcher = mock.patch('cinder.quota_utils.get_project_hierarchy', + self._get_project) + get_patcher.start() + self.addCleanup(get_patcher.stop) + + def _list_projects(context): + return self.project_by_id.values() + + list_patcher = mock.patch('cinder.quota_utils.get_all_projects', + _list_projects) + list_patcher.start() + self.addCleanup(list_patcher.stop) + self.auth_url = 'http://localhost:5000' self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') @@ -127,111 +150,24 @@ class QuotaSetsControllerTest(test.TestCase): def _get_project(self, context, id, subtree_as_ids=False): return self.project_by_id.get(id, self.FakeProject()) - @mock.patch('keystoneclient.client.Client') - @mock.patch('keystoneclient.session.Session') - def test_keystone_client_instantiation(self, ksclient_session, - ksclient_class): - context = self.req.environ['cinder.context'] - self.controller._keystone_client(context) - ksclient_class.assert_called_once_with(auth_url=self.auth_url, - session=ksclient_session()) - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v2(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v2.0' - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v2.0') - project = self.controller._get_project(context, context.project_id) - self.assertEqual(expected_project.__dict__, project.__dict__) - - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v3(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v3' - returned_project = self.FakeProject(context.project_id, 'bar') - del returned_project.subtree - keystoneclient.projects.get.return_value = returned_project - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v3', 'bar') - project = self.controller._get_project(context, context.project_id) - self.assertEqual(expected_project.__dict__, project.__dict__) - - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v3_with_subtree(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v3' - returned_project = self.FakeProject(context.project_id, 'bar') - subtree_dict = {'baz': {'quux': None}} - returned_project.subtree = subtree_dict - keystoneclient.projects.get.return_value = returned_project - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v3', 'bar', subtree_dict) - project = self.controller._get_project(context, context.project_id, - subtree_as_ids=True) - keystoneclient.projects.get.assert_called_once_with( - context.project_id, subtree_as_ids=True) - self.assertEqual(expected_project.__dict__, project.__dict__) +class QuotaSetsControllerTest(QuotaSetsControllerTestBase): + def setUp(self): + super(QuotaSetsControllerTest, self).setUp() + fixture = self.useFixture(config_fixture.Config(quota.CONF)) + fixture.config(quota_driver="cinder.quota.DbQuotaDriver") + quotas.QUOTAS = quota.VolumeTypeQuotaEngine() + self.controller = quotas.QuotaSetsController() def test_defaults(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result = self.controller.defaults(self.req, 'foo') self.assertDictMatch(make_body(), result) - def test_subproject_defaults(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - context = self.req.environ['cinder.context'] - context.project_id = self.B.id - result = self.controller.defaults(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - def test_show(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result) - def test_subproject_show(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.A.id - result = self.controller.show(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - - def test_subproject_show_in_hierarchy(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # An user scoped to a root project in an hierarchy can see its children - # quotas. - self.req.environ['cinder.context'].project_id = self.A.id - result = self.controller.show(self.req, self.D.id) - expected = make_subproject_body(tenant_id=self.D.id) - self.assertDictMatch(expected, result) - # An user scoped to a parent project can see its immediate children - # quotas. - self.req.environ['cinder.context'].project_id = self.B.id - result = self.controller.show(self.req, self.D.id) - expected = make_subproject_body(tenant_id=self.D.id) - self.assertDictMatch(expected, result) - - def test_subproject_show_target_project_equals_to_context_project(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.B.id - result = self.controller.show(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - def test_show_not_authorized(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self.req.environ['cinder.context'].is_admin = False self.req.environ['cinder.context'].user_id = 'bad_user' self.req.environ['cinder.context'].project_id = 'bad_project' @@ -239,29 +175,14 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo') def test_show_non_admin_user(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = exceptions.Forbidden self.controller._get_quotas = mock.Mock(side_effect= self.controller._get_quotas) result = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result) self.controller._get_quotas.assert_called_with( - self.req.environ['cinder.context'], 'foo', False, - parent_project_id=None) - - def test_subproject_show_not_authorized(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.B.id - self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, - self.req, self.C.id) - self.req.environ['cinder.context'].project_id = self.B.id - self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, - self.req, self.A.id) + self.req.environ['cinder.context'], 'foo', False) def test_update(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = make_body(gigabytes=2000, snapshots=15, volumes=5, backups=5, tenant_id=None) result = self.controller.update(self.req, 'foo', body) @@ -271,65 +192,9 @@ class QuotaSetsControllerTest(test.TestCase): result = self.controller.update(self.req, 'foo', body) self.assertDictMatch(body, result) - def test_update_subproject(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Update the quota of B to be equal to its parent quota - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, body) - self.assertDictMatch(body, result) - # Try to update the quota of C, it will not be allowed, since the - # project A doesn't have free quota available. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, - self.req, self.C.id, body) - # Successfully update the quota of D. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=1000, snapshots=7, - volumes=3, backups=3, tenant_id=None) - result = self.controller.update(self.req, self.D.id, body) - self.assertDictMatch(body, result) - # An admin of B can also update the quota of D, since D is its an - # immediate child. - self.req.environ['cinder.context'].project_id = self.B.id - body = make_body(gigabytes=1500, snapshots=10, - volumes=4, backups=4, tenant_id=None) - result = self.controller.update(self.req, self.D.id, body) - - def test_update_subproject_repetitive(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A volumes quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=10, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Update the quota of B to be equal to its parent quota - # three times should be successful, the quota will not be - # allocated to 'allocated' value of parent project - for i in range(0, 3): - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=10, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, body) - self.assertDictMatch(body, result) - - def test_update_subproject_not_in_hierarchy(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - - # Create another project hierarchy + def test_update_subproject_not_in_hierarchy_non_nested(self): + # When not using nested quotas, the hierarchy should not be considered + # for an update E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) F = self.FakeProject(id=uuid.uuid4().hex, parent_id=E.id) E.subtree = {F.id: F.subtree} @@ -342,50 +207,19 @@ class QuotaSetsControllerTest(test.TestCase): volumes=5, backups=5, tenant_id=None) result = self.controller.update(self.req, self.A.id, body) self.assertDictMatch(body, result) - # Try to update the quota of F, it will not be allowed, since the - # project E doesn't belongs to the project hierarchy of A. + # Try to update the quota of F, it will be allowed even though + # project E doesn't belong to the project hierarchy of A, because + # we are NOT using the nested quota driver self.req.environ['cinder.context'].project_id = self.A.id body = make_body(gigabytes=2000, snapshots=15, volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, - self.req, F.id, body) - - def test_update_subproject_with_not_root_context_project(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Try to update the quota of B, it will not be allowed, since the - # project in the context (B) is not a root project. - self.req.environ['cinder.context'].project_id = self.B.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, - self.req, self.B.id, body) - - def test_update_subproject_quota_when_parent_has_default_quotas(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Since the quotas of the project A were not updated, it will have - # default quotas. - self.req.environ['cinder.context'].project_id = self.A.id - # Update the project B quota. - expected = make_body(gigabytes=1000, snapshots=10, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, expected) - self.assertDictMatch(expected, result) + self.controller.update(self.req, F.id, body) @mock.patch( 'cinder.api.openstack.wsgi.Controller.validate_string_length') @mock.patch( 'cinder.api.openstack.wsgi.Controller.validate_integer') def test_update_limit(self, mock_validate_integer, mock_validate): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project mock_validate_integer.return_value = 10 body = {'quota_set': {'volumes': 10}} @@ -401,22 +235,16 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_invalid_value_key_value(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': "should_be_int"}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) def test_update_invalid_type_key_value(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': None}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) def test_update_multi_value_with_bad_data(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project orig_quota = self.controller.show(self.req, 'foo') body = make_body(gigabytes=2000, snapshots=15, volumes="should_be_int", backups=5, tenant_id=None) @@ -427,8 +255,6 @@ class QuotaSetsControllerTest(test.TestCase): self.assertDictMatch(orig_quota, new_quota) def test_update_bad_quota_limit(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': -1000}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) @@ -437,8 +263,6 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_no_admin(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self.req.environ['cinder.context'].is_admin = False self.req.environ['cinder.context'].project_id = 'foo' self.req.environ['cinder.context'].user_id = 'foo_user' @@ -468,8 +292,6 @@ class QuotaSetsControllerTest(test.TestCase): db.quota_usage_get_all_by_project(ctxt, 'foo')) def test_update_lower_than_existing_resources_when_skip_false(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}, 'skip_validation': 'false'} @@ -481,8 +303,6 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_lower_than_existing_resources_when_skip_true(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}, 'skip_validation': 'true'} @@ -491,8 +311,6 @@ class QuotaSetsControllerTest(test.TestCase): result['quota_set']['volumes']) def test_update_lower_than_existing_resources_without_skip_argument(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}} result = self.controller.update(self.req, 'foo', body) @@ -500,8 +318,6 @@ class QuotaSetsControllerTest(test.TestCase): result['quota_set']['volumes']) def test_delete(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result_show = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result_show) @@ -516,9 +332,7 @@ class QuotaSetsControllerTest(test.TestCase): result_show_after = self.controller.show(self.req, 'foo') self.assertDictMatch(result_show, result_show_after) - def test_subproject_delete(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project + def test_delete_with_allocated_quota_different_from_zero(self): self.req.environ['cinder.context'].project_id = self.A.id body = make_body(gigabytes=2000, snapshots=15, @@ -539,6 +353,365 @@ class QuotaSetsControllerTest(test.TestCase): result_show_after = self.controller.show(self.req, self.A.id) self.assertDictMatch(result_show, result_show_after) + def test_delete_no_admin(self): + self.req.environ['cinder.context'].is_admin = False + self.assertRaises(webob.exc.HTTPForbidden, self.controller.delete, + self.req, 'foo') + + def test_subproject_show_not_using_nested_quotas(self): + # Current roles say for non-nested quotas, an admin should be able to + # see anyones quota + self.req.environ['cinder.context'].project_id = self.B.id + self.controller.show(self.req, self.C.id) + self.controller.show(self.req, self.A.id) + + +class QuotaSetControllerValidateNestedQuotaSetup(QuotaSetsControllerTestBase): + """Validates the setup before using NestedQuota driver. + + Test case validates flipping on NestedQuota driver after using the + non-nested quota driver for some time. + """ + + def _create_project_hierarchy(self): + """Sets an environment used for nested quotas tests. + + Create a project hierarchy such as follows: + +-----------------+ + | | + | A G E | + | / \ \ | + | B C F | + | / | + | D | + +-----------------+ + """ + super(QuotaSetControllerValidateNestedQuotaSetup, + self)._create_project_hierarchy() + # Project A, B, C, D are already defined by parent test class + self.E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + self.F = self.FakeProject(id=uuid.uuid4().hex, parent_id=self.E.id) + self.G = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + + self.E.subtree = {self.F.id: self.F.subtree} + + self.project_by_id.update({self.E.id: self.E, self.F.id: self.F, + self.G.id: self.G}) + + def test_validate_nested_quotas_no_in_use_vols(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota = {'volumes': 5} + body = {'quota_set': quota} + self.controller.update(self.req, self.A.id, body) + + quota['volumes'] = 3 + self.controller.update(self.req, self.B.id, body) + # Allocated value for quota A is borked, because update was done + # without nested quota driver + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + # Fix the allocated values in DB + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use( + self.req) + + self.req.params['fix_allocated_quotas'] = False + # Ensure that we've properly fixed the allocated quotas + self.controller.validate_setup_for_nested_quota_use(self.req) + + # Over-allocate the quotas between children + self.controller.update(self.req, self.C.id, body) + + # This is we should fail because the child limits are too big + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + quota['volumes'] = 1 + self.controller.update(self.req, self.C.id, body) + + # Make sure we're validating all hierarchy trees + self.req.environ['cinder.context'].project_id = self.E.id + quota['volumes'] = 1 + self.controller.update(self.req, self.E.id, body) + quota['volumes'] = 3 + self.controller.update(self.req, self.F.id, body) + + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + # Put quotas in a good state + quota['volumes'] = 1 + self.controller.update(self.req, self.F.id, body) + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + + def _fake_quota_usage_get_all_by_project(self, context, project_id): + proj_vals = { + self.A.id: {'in_use': 1}, + self.B.id: {'in_use': 1}, + self.D.id: {'in_use': 0}, + self.C.id: {'in_use': 3}, + self.E.id: {'in_use': 0}, + self.F.id: {'in_use': 0}, + self.G.id: {'in_use': 0}, + } + return {'volumes': proj_vals[project_id]} + + @mock.patch('cinder.db.quota_usage_get_all_by_project') + def test_validate_nested_quotas_in_use_vols(self, mock_usage): + mock_usage.side_effect = self._fake_quota_usage_get_all_by_project + + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': 7} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + quota_limit['volumes'] = 3 + self.controller.update(self.req, self.B.id, body) + + quota_limit['volumes'] = 3 + self.controller.update(self.req, self.C.id, body) + + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + + quota_limit['volumes'] = 6 + self.controller.update(self.req, self.A.id, body) + + # Should fail because the one in_use volume of 'A' + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + @mock.patch('cinder.db.quota_usage_get_all_by_project') + def test_validate_nested_quotas_quota_borked(self, mock_usage): + mock_usage.side_effect = self._fake_quota_usage_get_all_by_project + + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': 7} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + # Other quotas would default to 0 but already have some limit being + # used + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + def test_validate_nested_quota_negative_limits(self): + # When we're validating, update the allocated values since we've + # been updating child limits + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': -1} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + quota_limit['volumes'] = 4 + self.controller.update(self.req, self.B.id, body) + + self.controller.validate_setup_for_nested_quota_use(self.req) + + quota_limit['volumes'] = -1 + self.controller.update(self.req, self.F.id, body) + # Should not work because can't have a child with negative limits + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + +class QuotaSetsControllerNestedQuotasTest(QuotaSetsControllerTestBase): + def setUp(self): + super(QuotaSetsControllerNestedQuotasTest, self).setUp() + fixture = self.useFixture(config_fixture.Config(quota.CONF)) + fixture.config(quota_driver="cinder.quota.NestedDbQuotaDriver") + quotas.QUOTAS = quota.VolumeTypeQuotaEngine() + self.controller = quotas.QuotaSetsController() + + def test_subproject_defaults(self): + context = self.req.environ['cinder.context'] + context.project_id = self.B.id + result = self.controller.defaults(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show(self): + self.req.environ['cinder.context'].project_id = self.A.id + result = self.controller.show(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_in_hierarchy(self): + # A user scoped to a root project in a hierarchy can see its children + # quotas. + self.req.environ['cinder.context'].project_id = self.A.id + result = self.controller.show(self.req, self.D.id) + expected = make_subproject_body(tenant_id=self.D.id) + self.assertDictMatch(expected, result) + # A user scoped to a parent project can see its immediate children + # quotas. + self.req.environ['cinder.context'].project_id = self.B.id + result = self.controller.show(self.req, self.D.id) + expected = make_subproject_body(tenant_id=self.D.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_target_project_equals_to_context_project( + self): + self.req.environ['cinder.context'].project_id = self.B.id + result = self.controller.show(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_not_authorized(self): + self.req.environ['cinder.context'].project_id = self.B.id + self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, + self.req, self.C.id) + self.req.environ['cinder.context'].project_id = self.B.id + self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, + self.req, self.A.id) + + def test_update_subproject_not_in_hierarchy(self): + + # Create another project hierarchy + E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + F = self.FakeProject(id=uuid.uuid4().hex, parent_id=E.id) + E.subtree = {F.id: F.subtree} + self.project_by_id[E.id] = E + self.project_by_id[F.id] = F + + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Try to update the quota of F, it will not be allowed, since the + # project E doesn't belongs to the project hierarchy of A. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.update, self.req, F.id, body) + + def test_update_subproject(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Update the quota of B to be equal to its parent quota + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None, is_child=True) + result = self.controller.update(self.req, self.B.id, body) + self.assertDictMatch(body, result) + # Try to update the quota of C, it will not be allowed, since the + # project A doesn't have free quota available. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None, is_child=True) + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, self.C.id, body) + # Successfully update the quota of D. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=1000, snapshots=7, + volumes=3, backups=3, tenant_id=None, is_child=True) + result = self.controller.update(self.req, self.D.id, body) + self.assertDictMatch(body, result) + # An admin of B can also update the quota of D, since D is its + # immediate child. + self.req.environ['cinder.context'].project_id = self.B.id + body = make_body(gigabytes=1500, snapshots=10, + volumes=4, backups=4, tenant_id=None, is_child=True) + self.controller.update(self.req, self.D.id, body) + + def test_update_subproject_negative_limit(self): + # Should not be able to set a negative limit for a child project (will + # require further fixes to allow for this) + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(volumes=-1, is_child=True) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, self.req, self.B.id, body) + + def test_update_subproject_repetitive(self): + # Update the project A volumes quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=10, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Update the quota of B to be equal to its parent quota + # three times should be successful, the quota will not be + # allocated to 'allocated' value of parent project + for i in range(0, 3): + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=10, backups=5, tenant_id=None, + is_child=True) + result = self.controller.update(self.req, self.B.id, body) + self.assertDictMatch(body, result) + + def test_update_subproject_with_not_root_context_project(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Try to update the quota of B, it will not be allowed, since the + # project in the context (B) is not a root project. + self.req.environ['cinder.context'].project_id = self.B.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, + self.req, self.B.id, body) + + def test_update_subproject_quota_when_parent_has_default_quotas(self): + # Since the quotas of the project A were not updated, it will have + # default quotas. + self.req.environ['cinder.context'].project_id = self.A.id + # Update the project B quota. + expected = make_body(gigabytes=1000, snapshots=10, + volumes=5, backups=5, tenant_id=None, + is_child=True) + result = self.controller.update(self.req, self.B.id, expected) + self.assertDictMatch(expected, result) + + def test_subproject_delete(self): + self.req.environ['cinder.context'].project_id = self.A.id + + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, + backup_gigabytes=1000, tenant_id=None, is_child=True) + result_update = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result_update) + + # Set usage param to True in order to see get allocated values. + self.req.params = {'usage': 'True'} + result_show = self.controller.show(self.req, self.A.id) + + result_update = self.controller.update(self.req, self.B.id, body) + self.assertDictMatch(body, result_update) + + self.controller.delete(self.req, self.B.id) + + result_show_after = self.controller.show(self.req, self.A.id) + self.assertDictMatch(result_show, result_show_after) + def test_subproject_delete_not_considering_default_quotas(self): """Test delete subprojects' quotas won't consider default quotas. @@ -551,8 +724,6 @@ class QuotaSetsControllerTest(test.TestCase): updating the allocated values of the parent project. Thus, the delete operation should succeed. """ - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self.req.environ['cinder.context'].project_id = self.A.id body = {'quota_set': {'volumes': 5}} @@ -567,35 +738,20 @@ class QuotaSetsControllerTest(test.TestCase): self.controller.delete(self.req, self.B.id) - def test_delete_with_allocated_quota_different_from_zero(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project + def test_subproject_delete_with_child_present(self): + # Update the project A quota. self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(volumes=5) + self.controller.update(self.req, self.A.id, body) - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, - backup_gigabytes=1000, tenant_id=None) - result_update = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result_update) + # Allocate some of that quota to a child project + body = make_body(volumes=3, is_child=True) + self.controller.update(self.req, self.B.id, body) - # Set usage param to True in order to see get allocated values. - self.req.params = {'usage': 'True'} - result_show = self.controller.show(self.req, self.A.id) - - result_update = self.controller.update(self.req, self.B.id, body) - self.assertDictMatch(body, result_update) - - self.controller.delete(self.req, self.B.id) - - result_show_after = self.controller.show(self.req, self.A.id) - self.assertDictMatch(result_show, result_show_after) - - def test_delete_no_admin(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].is_admin = False - self.assertRaises(webob.exc.HTTPForbidden, self.controller.delete, - self.req, 'foo') + # Deleting 'A' should be disallowed since 'B' is using some of that + # quota + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.delete, + self.req, self.A.id) class QuotaSerializerTest(test.TestCase): diff --git a/cinder/tests/unit/test_quota.py b/cinder/tests/unit/test_quota.py index 11f792d44da..d90f102356a 100644 --- a/cinder/tests/unit/test_quota.py +++ b/cinder/tests/unit/test_quota.py @@ -20,6 +20,7 @@ import datetime import mock from oslo_config import cfg +from oslo_config import fixture as config_fixture from oslo_utils import timeutils import six @@ -37,6 +38,8 @@ from cinder import test import cinder.tests.unit.image.fake from cinder import volume +from keystonemiddleware import auth_token + CONF = cfg.CONF @@ -358,11 +361,9 @@ class FakeDriver(object): return resources def get_project_quotas(self, context, resources, project_id, - quota_class=None, defaults=True, usages=True, - parent_project_id=None): + quota_class=None, defaults=True, usages=True): self.called.append(('get_project_quotas', context, resources, - project_id, quota_class, defaults, usages, - parent_project_id)) + project_id, quota_class, defaults, usages)) return resources def limit_check(self, context, resources, values, project_id=None): @@ -613,7 +614,6 @@ class QuotaEngineTestCase(test.TestCase): def test_get_project_quotas(self): context = FakeContext(None, None) driver = FakeDriver() - parent_project_id = None quota_obj = self._make_quota_obj(driver) result1 = quota_obj.get_project_quotas(context, 'test_project') result2 = quota_obj.get_project_quotas(context, 'test_project', @@ -628,33 +628,26 @@ class QuotaEngineTestCase(test.TestCase): 'test_project', None, True, - True, - parent_project_id), + True), ('get_project_quotas', context, quota_obj.resources, 'test_project', 'test_class', False, - False, - parent_project_id), ], driver.called) + False), ], driver.called) self.assertEqual(quota_obj.resources, result1) self.assertEqual(quota_obj.resources, result2) def test_get_subproject_quotas(self): context = FakeContext(None, None) driver = FakeDriver() - parent_project_id = 'test_parent_project_id' quota_obj = self._make_quota_obj(driver) - result1 = quota_obj.get_project_quotas(context, 'test_project', - parent_project_id= - parent_project_id) + result1 = quota_obj.get_project_quotas(context, 'test_project') result2 = quota_obj.get_project_quotas(context, 'test_project', quota_class='test_class', defaults=False, - usages=False, - parent_project_id= - parent_project_id) + usages=False) self.assertEqual([ ('get_project_quotas', @@ -663,16 +656,14 @@ class QuotaEngineTestCase(test.TestCase): 'test_project', None, True, - True, - parent_project_id), + True), ('get_project_quotas', context, quota_obj.resources, 'test_project', 'test_class', False, - False, - parent_project_id), ], driver.called) + False), ], driver.called) self.assertEqual(quota_obj.resources, result1) self.assertEqual(quota_obj.resources, result2) @@ -886,9 +877,9 @@ class VolumeTypeQuotaEngineTestCase(test.TestCase): db.volume_type_destroy(ctx, vtype2['id']) -class DbQuotaDriverTestCase(test.TestCase): +class DbQuotaDriverBaseTestCase(test.TestCase): def setUp(self): - super(DbQuotaDriverTestCase, self).setUp() + super(DbQuotaDriverBaseTestCase, self).setUp() self.flags(quota_volumes=10, quota_snapshots=10, @@ -900,7 +891,21 @@ class DbQuotaDriverTestCase(test.TestCase): max_age=0, ) - self.driver = quota.DbQuotaDriver() + # These can be used for expected defaults for child/non-child + self._default_quotas_non_child = dict( + volumes=10, + snapshots=10, + gigabytes=1000, + backups=10, + backup_gigabytes=1000, + per_volume_gigabytes=-1) + self._default_quotas_child = dict( + volumes=0, + snapshots=0, + gigabytes=0, + backups=0, + backup_gigabytes=0, + per_volume_gigabytes=0) self.calls = [] @@ -909,38 +914,6 @@ class DbQuotaDriverTestCase(test.TestCase): self.mock_utcnow = patcher.start() self.mock_utcnow.return_value = datetime.datetime.utcnow() - def test_get_defaults(self): - # Use our pre-defined resources - self._stub_quota_class_get_default() - self._stub_volume_type_get_all() - result = self.driver.get_defaults(None, quota.QUOTAS.resources) - - self.assertEqual( - dict( - volumes=10, - snapshots=10, - gigabytes=1000, - backups=10, - backup_gigabytes=1000, - per_volume_gigabytes=-1), result) - - def test_subproject_get_defaults(self): - # Test subproject default values. - self._stub_volume_type_get_all() - parent_project_id = 'test_parent_project_id' - result = self.driver.get_defaults(None, - quota.QUOTAS.resources, - parent_project_id) - - self.assertEqual( - dict( - volumes=0, - snapshots=0, - gigabytes=0, - backups=0, - backup_gigabytes=0, - per_volume_gigabytes=0), result) - def _stub_quota_class_get_default(self): # Stub out quota_class_get_default def fake_qcgd(context): @@ -967,6 +940,37 @@ class DbQuotaDriverTestCase(test.TestCase): backup_gigabytes=500) self.stubs.Set(db, 'quota_class_get_all_by_name', fake_qcgabn) + def _stub_allocated_get_all_by_project(self, allocated_quota=False): + def fake_qagabp(context, project_id): + self.calls.append('quota_allocated_get_all_by_project') + if allocated_quota: + return dict(project_id=project_id, volumes=3) + return dict(project_id=project_id) + + self.stubs.Set(db, 'quota_allocated_get_all_by_project', fake_qagabp) + + +class DbQuotaDriverTestCase(DbQuotaDriverBaseTestCase): + def setUp(self): + super(DbQuotaDriverTestCase, self).setUp() + + self.driver = quota.DbQuotaDriver() + + def test_get_defaults(self): + # Use our pre-defined resources + self._stub_quota_class_get_default() + self._stub_volume_type_get_all() + result = self.driver.get_defaults(None, quota.QUOTAS.resources) + + self.assertEqual( + dict( + volumes=10, + snapshots=10, + gigabytes=1000, + backups=10, + backup_gigabytes=1000, + per_volume_gigabytes=-1), result) + def test_get_class_quotas(self): self._stub_quota_class_get_all_by_name() self._stub_volume_type_get_all() @@ -1017,33 +1021,6 @@ class DbQuotaDriverTestCase(test.TestCase): self._stub_quota_class_get_all_by_name() self._stub_quota_class_get_default() - def _stub_get_by_subproject(self): - def fake_qgabp(context, project_id): - self.calls.append('quota_get_all_by_project') - self.assertEqual('test_project', project_id) - return dict(volumes=10, gigabytes=50, reserved=0) - - def fake_qugabp(context, project_id): - self.calls.append('quota_usage_get_all_by_project') - self.assertEqual('test_project', project_id) - return dict(volumes=dict(in_use=2, reserved=0), - gigabytes=dict(in_use=10, reserved=0)) - - self.stubs.Set(db, 'quota_get_all_by_project', fake_qgabp) - self.stubs.Set(db, 'quota_usage_get_all_by_project', fake_qugabp) - - self._stub_quota_class_get_all_by_name() - - def _stub_allocated_get_all_by_project(self, allocated_quota=False): - def fake_qagabp(context, project_id): - self.calls.append('quota_allocated_get_all_by_project') - self.assertEqual('test_project', project_id) - if allocated_quota: - return dict(project_id=project_id, volumes=3) - return dict(project_id=project_id) - - self.stubs.Set(db, 'quota_allocated_get_all_by_project', fake_qagabp) - def test_get_project_quotas(self): self._stub_get_by_project() self._stub_volume_type_get_all() @@ -1115,45 +1092,6 @@ class DbQuotaDriverTestCase(test.TestCase): allocated=0) ), result) - def test_get_subproject_quotas(self): - self._stub_get_by_subproject() - self._stub_volume_type_get_all() - self._stub_allocated_get_all_by_project(allocated_quota=True) - parent_project_id = 'test_parent_project_id' - result = self.driver.get_project_quotas( - FakeContext('test_project', None), - quota.QUOTAS.resources, 'test_project', - parent_project_id=parent_project_id) - - self.assertEqual(['quota_get_all_by_project', - 'quota_usage_get_all_by_project', - 'quota_allocated_get_all_by_project', ], self.calls) - self.assertEqual(dict(volumes=dict(limit=10, - in_use=2, - reserved=0, - allocated=3, ), - snapshots=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - gigabytes=dict(limit=50, - in_use=10, - reserved=0, - allocated=0, ), - backups=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - backup_gigabytes=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - per_volume_gigabytes=dict(in_use=0, - limit=0, - reserved=0, - allocated=0) - ), result) - def test_get_project_quotas_alt_context_no_class(self): self._stub_get_by_project() self._stub_volume_type_get_all() @@ -1423,6 +1361,231 @@ class DbQuotaDriverTestCase(test.TestCase): self.calls) +class NestedDbQuotaDriverBaseTestCase(DbQuotaDriverBaseTestCase): + def setUp(self): + super(NestedDbQuotaDriverBaseTestCase, self).setUp() + self.context = context.RequestContext('user_id', + 'project_id', + is_admin=True, + auth_token="fake_token") + self.auth_url = 'http://localhost:5000' + self._child_proj_id = 'child_id' + self._non_child_proj_id = 'non_child_id' + + keystone_mock = mock.Mock() + keystone_mock.version = 'v3' + + class FakeProject(object): + def __init__(self, parent_id): + self.parent_id = parent_id + + def fake_get_project(project_id, subtree_as_ids=False): + # Enable imitation of projects with and without parents + if project_id == self._child_proj_id: + return FakeProject('parent_id') + else: + return FakeProject(None) + + keystone_mock.projects.get.side_effect = fake_get_project + + def _keystone_mock(self): + return keystone_mock + + keystone_patcher = mock.patch('cinder.quota_utils._keystone_client', + _keystone_mock) + keystone_patcher.start() + self.addCleanup(keystone_patcher.stop) + + self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) + self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') + self.driver = quota.NestedDbQuotaDriver() + + def _stub_get_by_subproject(self): + def fake_qgabp(context, project_id): + self.calls.append('quota_get_all_by_project') + return dict(volumes=10, gigabytes=50, reserved=0) + + def fake_qugabp(context, project_id): + self.calls.append('quota_usage_get_all_by_project') + return dict(volumes=dict(in_use=2, reserved=0), + gigabytes=dict(in_use=10, reserved=0)) + + self.stubs.Set(db, 'quota_get_all_by_project', fake_qgabp) + self.stubs.Set(db, 'quota_usage_get_all_by_project', fake_qugabp) + + self._stub_quota_class_get_all_by_name() + + +class NestedDbQuotaDriverTestCase(NestedDbQuotaDriverBaseTestCase): + def test_get_defaults(self): + self._stub_volume_type_get_all() + + # Test for child project defaults + result = self.driver.get_defaults(self.context, + quota.QUOTAS.resources, + self._child_proj_id) + self.assertEqual(self._default_quotas_child, result) + + # Test for non-child project defaults + result = self.driver.get_defaults(self.context, + quota.QUOTAS.resources, + self._non_child_proj_id) + self.assertEqual(self._default_quotas_non_child, result) + + def test_subproject_enforce_defaults(self): + # Non-child defaults should allow volume to get created + self.driver.reserve(self.context, + quota.QUOTAS.resources, + {'volumes': 1, 'gigabytes': 1}, + project_id=self._non_child_proj_id) + + # Child defaults should not allow volume to be created + self.assertRaises(exception.OverQuota, + self.driver.reserve, self.context, + quota.QUOTAS.resources, + {'volumes': 1, 'gigabytes': 1}, + project_id=self._child_proj_id) + + def test_get_subproject_quotas(self): + self._stub_get_by_subproject() + self._stub_volume_type_get_all() + self._stub_allocated_get_all_by_project(allocated_quota=True) + result = self.driver.get_project_quotas( + self.context, + quota.QUOTAS.resources, self._child_proj_id) + + self.assertEqual(['quota_get_all_by_project', + 'quota_usage_get_all_by_project', + 'quota_allocated_get_all_by_project', ], self.calls) + self.assertEqual(dict(volumes=dict(limit=10, + in_use=2, + reserved=0, + allocated=3, ), + snapshots=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + gigabytes=dict(limit=50, + in_use=10, + reserved=0, + allocated=0, ), + backups=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + backup_gigabytes=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + per_volume_gigabytes=dict(in_use=0, + limit=0, + reserved=0, + allocated=0) + ), result) + + +class NestedQuotaValidation(NestedDbQuotaDriverBaseTestCase): + def setUp(self): + super(NestedQuotaValidation, self).setUp() + """ + Quota hierarchy setup like so + +-----------+ + | | + | A | + | / \ | + | B C | + | / | + | D | + +-----------+ + """ + self.project_tree = {'A': {'B': {'D': None}, 'C': None}} + self.proj_vals = { + 'A': {'limit': 7, 'in_use': 1, 'alloc': 6}, + 'B': {'limit': 3, 'in_use': 1, 'alloc': 2}, + 'D': {'limit': 2, 'in_use': 0}, + 'C': {'limit': 3, 'in_use': 3}, + } + + # Just using one resource currently for simplicity of test + self.resources = {'volumes': quota.ReservableResource( + 'volumes', '_sync_volumes', 'quota_volumes')} + + to_patch = [('cinder.db.quota_allocated_get_all_by_project', + self._fake_quota_allocated_get_all_by_project), + ('cinder.db.quota_get_all_by_project', + self._fake_quota_get_all_by_project), + ('cinder.db.quota_usage_get_all_by_project', + self._fake_quota_usage_get_all_by_project)] + + for patch_path, patch_obj in to_patch: + patcher = mock.patch(patch_path, patch_obj) + patcher.start() + self.addCleanup(patcher.stop) + + def _fake_quota_get_all_by_project(self, context, project_id): + return {'volumes': self.proj_vals[project_id]['limit']} + + def _fake_quota_usage_get_all_by_project(self, context, project_id): + return {'volumes': self.proj_vals[project_id]} + + def _fake_quota_allocated_get_all_by_project(self, context, project_id): + ret = {'project_id': project_id} + proj_val = self.proj_vals[project_id] + if 'alloc' in proj_val: + ret['volumes'] = proj_val['alloc'] + return ret + + def test_validate_nested_quotas(self): + self.driver.validate_nested_setup(self.context, + self.resources, self.project_tree) + + # Fail because 7 - 2 < 3 + 3 + self.proj_vals['A']['in_use'] = 2 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['A']['in_use'] = 1 + + # Fail because 7 - 1 < 3 + 7 + self.proj_vals['C']['limit'] = 7 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['C']['limit'] = 3 + + # Fail because 3 < 4 + self.proj_vals['D']['limit'] = 4 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['D']['limit'] = 2 + + def test_validate_nested_quotas_negative_child_limit(self): + self.proj_vals['B']['limit'] = -1 + self.assertRaises( + exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + def test_validate_nested_quotas_usage_over_limit(self): + + self.proj_vals['D']['in_use'] = 5 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + def test_validate_nested_quota_bad_allocated_quotas(self): + + self.proj_vals['A']['alloc'] = 5 + self.proj_vals['B']['alloc'] = 8 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + class FakeSession(object): def begin(self): return self diff --git a/cinder/tests/unit/test_quota_utils.py b/cinder/tests/unit/test_quota_utils.py new file mode 100644 index 00000000000..c220651e3a4 --- /dev/null +++ b/cinder/tests/unit/test_quota_utils.py @@ -0,0 +1,110 @@ +# Copyright 2016 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from cinder import context +from cinder import exception +from cinder import quota_utils +from cinder import test + +from keystoneclient import exceptions +from keystonemiddleware import auth_token + +from oslo_config import cfg +from oslo_config import fixture as config_fixture + +CONF = cfg.CONF + + +class QuotaUtilsTest(test.TestCase): + class FakeProject(object): + def __init__(self, id='foo', parent_id=None): + self.id = id + self.parent_id = parent_id + self.subtree = None + + def setUp(self): + super(QuotaUtilsTest, self).setUp() + + self.auth_url = 'http://localhost:5000' + self.context = context.RequestContext('fake_user', 'fake_proj_id') + self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) + self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') + + @mock.patch('keystoneclient.client.Client') + @mock.patch('keystoneclient.session.Session') + def test_keystone_client_instantiation(self, ksclient_session, + ksclient_class): + quota_utils._keystone_client(self.context) + ksclient_class.assert_called_once_with(auth_url=self.auth_url, + session=ksclient_session(), + version=(3, 0)) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v2(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v2.0' + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v2.0') + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v3(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v3' + returned_project = self.FakeProject(self.context.project_id, 'bar') + del returned_project.subtree + keystoneclient.projects.get.return_value = returned_project + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v3', 'bar') + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v3_with_subtree(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v3' + returned_project = self.FakeProject(self.context.project_id, 'bar') + subtree_dict = {'baz': {'quux': None}} + returned_project.subtree = subtree_dict + keystoneclient.projects.get.return_value = returned_project + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v3', 'bar', subtree_dict) + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id, subtree_as_ids=True) + keystoneclient.projects.get.assert_called_once_with( + self.context.project_id, subtree_as_ids=True) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('cinder.quota_utils._keystone_client') + def test_validate_nested_projects_with_keystone_v2(self, _keystone_client): + _keystone_client.side_effect = exceptions.VersionNotAvailable + + self.assertRaises(exception.CinderException, + quota_utils.validate_setup_for_nested_quota_use, + self.context, [], None) + + @mock.patch('cinder.quota_utils._keystone_client') + def test_validate_nested_projects_non_cloud_admin(self, _keystone_client): + # Covers not cloud admin or using old policy.json + _keystone_client.side_effect = exceptions.Forbidden + + self.assertRaises(exception.CinderException, + quota_utils.validate_setup_for_nested_quota_use, + self.context, [], None) diff --git a/etc/cinder/policy.json b/etc/cinder/policy.json index 47d1cadbdeb..9e280cc6409 100644 --- a/etc/cinder/policy.json +++ b/etc/cinder/policy.json @@ -39,6 +39,7 @@ "volume_extension:quotas:update": "rule:admin_api", "volume_extension:quotas:delete": "rule:admin_api", "volume_extension:quota_classes": "rule:admin_api", + "volume_extension:quota_classes:validate_setup_for_nested_quota_use": "rule: admin_api", "volume_extension:volume_admin_actions:reset_status": "rule:admin_api", "volume_extension:snapshot_admin_actions:reset_status": "rule:admin_api", diff --git a/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml b/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml new file mode 100644 index 00000000000..43a33a3dff5 --- /dev/null +++ b/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml @@ -0,0 +1,12 @@ +--- +features: + - Split nested quota support into a separate driver. In + order to use nested quotas, change the following config + ``quota_driver = cinder.quota.NestedDbQuotaDriver`` after + running the following admin API + "os-quota-sets/validate_setup_for_nested_quota_use" command + to ensure the existing quota values make sense to nest. +upgrade: + - Nested quotas will no longer be used by default, but can be + configured by setting + ``quota_driver = cinder.quota.NestedDbQuotaDriver``