From 54ee1b2aaa87c4d40db6c9fb35b93a6186effed2 Mon Sep 17 00:00:00 2001 From: "Andrei V. Ostapenko" Date: Fri, 11 Oct 2013 18:24:07 +0300 Subject: [PATCH] Added per user-tenant quota support Added per user-tenant quota support. Added user-quotas extension, that turns on user quota support if it is loaded. Added force parameter, that lets to ignore check if admin want to force update when run 'manila quota-update' Added 'extended-quotas' extension that has provides ability for admins to be able to delete a non-default quota (absolute limit) for a tenant, so that tenant's quota will revert back to the configured default, and makes the force parameter always be passed if the client wants to set the new quota lower than what is already used and reserved. Added user quota support to db.api, sqlalchemy.api, sqlalchemy.models. Added migrations for user quota support. Implement bp: user-quota-support Change-Id: Ifb8f8a041c2fa54e2ed3a8219e87607b161438ca --- manila/api/contrib/extended_quotas.py | 28 + manila/api/contrib/quotas.py | 165 ++++- manila/api/contrib/user_quotas.py | 27 + manila/api/extensions.py | 1 + manila/db/api.py | 108 ++-- manila/db/sqlalchemy/api.py | 579 ++++++++++++------ .../005_make_user_quotas_key_and_value.py | 88 +++ manila/db/sqlalchemy/models.py | 21 + manila/exception.py | 26 +- manila/quota.py | 463 ++++++++++---- manila/tests/test_quota.py | 370 +++++++++-- 11 files changed, 1460 insertions(+), 416 deletions(-) create mode 100644 manila/api/contrib/extended_quotas.py create mode 100644 manila/api/contrib/user_quotas.py create mode 100644 manila/db/sqlalchemy/migrate_repo/versions/005_make_user_quotas_key_and_value.py diff --git a/manila/api/contrib/extended_quotas.py b/manila/api/contrib/extended_quotas.py new file mode 100644 index 0000000000..e4971bc82e --- /dev/null +++ b/manila/api/contrib/extended_quotas.py @@ -0,0 +1,28 @@ +# Copyright 2013 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from manila.api import extensions + + +class Extended_quotas(extensions.ExtensionDescriptor): + """Adds ability for admins to delete quota + and optionally force the update Quota command. + """ + + name = "ExtendedQuotas" + alias = "os-extended-quotas" + namespace = ("http://docs.openstack.org/compute/ext/extended_quotas" + "/api/v1.1") + updated = "2013-06-09T00:00:00+00:00" diff --git a/manila/api/contrib/quotas.py b/manila/api/contrib/quotas.py index 9d3eb81187..dc4e25145e 100644 --- a/manila/api/contrib/quotas.py +++ b/manila/api/contrib/quotas.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import urlparse import webob from manila.api import extensions @@ -23,14 +24,20 @@ from manila.api import xmlutil from manila import db from manila.db.sqlalchemy import api as sqlalchemy_api from manila import exception +from manila.openstack.common.gettextutils import _ +from manila.openstack.common import log as logging +from manila.openstack.common import strutils from manila import quota QUOTAS = quota.QUOTAS +LOG = logging.getLogger(__name__) +NON_QUOTA_KEYS = ['tenant_id', 'id', 'force'] authorize_update = extensions.extension_authorizer('compute', 'quotas:update') authorize_show = extensions.extension_authorizer('compute', 'quotas:show') +authorize_delete = extensions.extension_authorizer('compute', 'quotas:delete') class QuotaTemplate(xmlutil.TemplateBuilder): @@ -47,6 +54,9 @@ class QuotaTemplate(xmlutil.TemplateBuilder): class QuotaSetsController(object): + def __init__(self, ext_mgr): + self.ext_mgr = ext_mgr + def _format_quota_set(self, project_id, quota_set): """Convert the quota object to a result dict""" @@ -57,14 +67,25 @@ class QuotaSetsController(object): return dict(quota_set=result) - def _validate_quota_limit(self, limit): + def _validate_quota_limit(self, limit, minimum, maximum, force_update): # NOTE: -1 is a flag value for unlimited if limit < -1: msg = _("Quota limit must be -1 or greater.") raise webob.exc.HTTPBadRequest(explanation=msg) + if ((limit < minimum and not force_update) and + (maximum != -1 or (maximum == -1 and limit != -1))): + msg = _("Quota limit must greater than %s.") % minimum + raise webob.exc.HTTPBadRequest(explanation=msg) + if maximum != -1 and limit > maximum: + msg = _("Quota limit must less than %s.") % maximum + raise webob.exc.HTTPBadRequest(explanation=msg) - def _get_quotas(self, context, id, usages=False): - values = QUOTAS.get_project_quotas(context, id, usages=usages) + def _get_quotas(self, context, id, user_id=None, usages=False): + if user_id: + values = QUOTAS.get_user_quotas(context, id, user_id, + usages=usages) + else: + values = QUOTAS.get_project_quotas(context, id, usages=usages) if usages: return values @@ -75,29 +96,120 @@ class QuotaSetsController(object): def show(self, req, id): context = req.environ['manila.context'] authorize_show(context) + params = urlparse.parse_qs(req.environ.get('QUERY_STRING', '')) + user_id = None + if self.ext_mgr.is_loaded('os-user-quotas'): + user_id = params.get('user_id', [None])[0] try: sqlalchemy_api.authorize_project_context(context, id) + return self._format_quota_set(id, + self._get_quotas(context, id, user_id=user_id)) except exception.NotAuthorized: raise webob.exc.HTTPForbidden() - return self._format_quota_set(id, self._get_quotas(context, id)) - @wsgi.serializers(xml=QuotaTemplate) def update(self, req, id, body): context = req.environ['manila.context'] authorize_update(context) project_id = id - for key in body['quota_set'].keys(): - if key in QUOTAS: - value = int(body['quota_set'][key]) - self._validate_quota_limit(value) + + bad_keys = [] + + # By default, we can force update the quota if the extended + # is not loaded + force_update = True + extended_loaded = False + if self.ext_mgr.is_loaded('os-extended-quotas'): + # force optional has been enabled, the default value of + # force_update need to be changed to False + extended_loaded = True + force_update = False + + user_id = None + if self.ext_mgr.is_loaded('os-user-quotas'): + # Update user quotas only if the extended is loaded + params = urlparse.parse_qs(req.environ.get('QUERY_STRING', '')) + user_id = params.get('user_id', [None])[0] + + try: + settable_quotas = QUOTAS.get_settable_quotas(context, project_id, + user_id=user_id) + except exception.NotAuthorized: + raise webob.exc.HTTPForbidden() + + for key, value in body['quota_set'].items(): + if (key not in QUOTAS and + key not in NON_QUOTA_KEYS): + bad_keys.append(key) + continue + if key == 'force' and extended_loaded: + # only check the force optional when the extended has + # been loaded + force_update = strutils.bool_from_string(value) + elif key not in NON_QUOTA_KEYS and value: try: - db.quota_update(context, project_id, key, value) - except exception.ProjectQuotaNotFound: - db.quota_create(context, project_id, key, value) - except exception.AdminRequired: - raise webob.exc.HTTPForbidden() - return {'quota_set': self._get_quotas(context, id)} + value = int(value) + except (ValueError, TypeError): + msg = _("Quota '%(value)s' for %(key)s should be " + "integer.") % {'value': value, 'key': key} + LOG.warn(msg) + raise webob.exc.HTTPBadRequest(explanation=msg) + + LOG.debug(_("force update quotas: %s") % force_update) + + if len(bad_keys) > 0: + msg = _("Bad key(s) %s in quota_set") % ",".join(bad_keys) + raise webob.exc.HTTPBadRequest(explanation=msg) + + try: + quotas = self._get_quotas(context, id, user_id=user_id, + usages=True) + except exception.NotAuthorized: + raise webob.exc.HTTPForbidden() + + for key, value in body['quota_set'].items(): + if key in NON_QUOTA_KEYS or (not value and value != 0): + continue + # validate whether already used and reserved exceeds the new + # quota, this check will be ignored if admin want to force + # update + try: + value = int(value) + except (ValueError, TypeError): + msg = _("Quota '%(value)s' for %(key)s should be " + "integer.") % {'value': value, 'key': key} + LOG.warn(msg) + raise webob.exc.HTTPBadRequest(explanation=msg) + + if force_update is not True and value >= 0: + quota_value = quotas.get(key) + if quota_value and quota_value['limit'] >= 0: + quota_used = (quota_value['in_use'] + + quota_value['reserved']) + LOG.debug(_("Quota %(key)s used: %(quota_used)s, " + "value: %(value)s."), + {'key': key, 'quota_used': quota_used, + 'value': value}) + if quota_used > value: + msg = (_("Quota value %(value)s for %(key)s are " + "greater than already used and reserved " + "%(quota_used)s") % + {'value': value, 'key': key, + 'quota_used': quota_used}) + raise webob.exc.HTTPBadRequest(explanation=msg) + + minimum = settable_quotas[key]['minimum'] + maximum = settable_quotas[key]['maximum'] + self._validate_quota_limit(value, minimum, maximum, force_update) + try: + db.quota_create(context, project_id, key, value, + user_id=user_id) + except exception.QuotaExists: + db.quota_update(context, project_id, key, value, + user_id=user_id) + except exception.AdminRequired: + raise webob.exc.HTTPForbidden() + return {'quota_set': self._get_quotas(context, id, user_id=user_id)} @wsgi.serializers(xml=QuotaTemplate) def defaults(self, req, id): @@ -105,6 +217,26 @@ class QuotaSetsController(object): authorize_show(context) return self._format_quota_set(id, QUOTAS.get_defaults(context)) + def delete(self, req, id): + if self.ext_mgr.is_loaded('os-extended-quotas'): + context = req.environ['manila.context'] + authorize_delete(context) + params = urlparse.parse_qs(req.environ.get('QUERY_STRING', '')) + user_id = params.get('user_id', [None])[0] + if user_id and not self.ext_mgr.is_loaded('os-user-quotas'): + raise webob.exc.HTTPNotFound() + try: + sqlalchemy_api.authorize_project_context(context, id) + if user_id: + QUOTAS.destroy_all_by_project_and_user(context, + id, user_id) + else: + QUOTAS.destroy_all_by_project(context, id) + return webob.Response(status_int=202) + except exception.NotAuthorized: + raise webob.exc.HTTPForbidden() + raise webob.exc.HTTPNotFound() + class Quotas(extensions.ExtensionDescriptor): """Quotas management support""" @@ -116,9 +248,8 @@ class Quotas(extensions.ExtensionDescriptor): def get_resources(self): resources = [] - res = extensions.ResourceExtension('os-quota-sets', - QuotaSetsController(), + QuotaSetsController(self.ext_mgr), member_actions={'defaults': 'GET'}) resources.append(res) diff --git a/manila/api/contrib/user_quotas.py b/manila/api/contrib/user_quotas.py new file mode 100644 index 0000000000..2119fbee0f --- /dev/null +++ b/manila/api/contrib/user_quotas.py @@ -0,0 +1,27 @@ +# Copyright 2013 OpenStack Foundation +# Author: Andrei Ostapenko +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from manila.api import extensions + + +class User_quotas(extensions.ExtensionDescriptor): + """Project user quota support.""" + + name = "UserQuotas" + alias = "os-user-quotas" + namespace = ("http://docs.openstack.org/compute/ext/user_quotas" + "/api/v1.1") + updated = "2013-07-18T00:00:00+00:00" diff --git a/manila/api/extensions.py b/manila/api/extensions.py index a4ac36fdcb..ee64221639 100644 --- a/manila/api/extensions.py +++ b/manila/api/extensions.py @@ -65,6 +65,7 @@ class ExtensionDescriptor(object): """Register extension with the extension manager.""" ext_mgr.register(self) + self.ext_mgr = ext_mgr def get_resources(self): """List of extensions.ResourceExtension extension objects. diff --git a/manila/db/api.py b/manila/db/api.py index f34abb2cc8..c89ccd7ea8 100644 --- a/manila/db/api.py +++ b/manila/db/api.py @@ -164,14 +164,20 @@ def migration_get_all_unconfirmed(context, confirm_window): #################### -def quota_create(context, project_id, resource, limit): +def quota_create(context, project_id, resource, limit, user_id=None): """Create a quota for the given project and resource.""" - return IMPL.quota_create(context, project_id, resource, limit) + return IMPL.quota_create(context, project_id, resource, limit, + user_id=user_id) -def quota_get(context, project_id, resource): +def quota_get(context, project_id, resource, user_id=None): """Retrieve a quota or raise if it does not exist.""" - return IMPL.quota_get(context, project_id, resource) + return IMPL.quota_get(context, project_id, resource, user_id=user_id) + + +def quota_get_all_by_project_and_user(context, project_id, user_id): + """Retrieve all quotas associated with a given project and user.""" + return IMPL.quota_get_all_by_project_and_user(context, project_id, user_id) def quota_get_all_by_project(context, project_id): @@ -179,14 +185,15 @@ def quota_get_all_by_project(context, project_id): return IMPL.quota_get_all_by_project(context, project_id) -def quota_update(context, project_id, resource, limit): +def quota_get_all(context, project_id): + """Retrieve all user quotas associated with a given project.""" + return IMPL.quota_get_all(context, project_id) + + +def quota_update(context, project_id, resource, limit, user_id=None): """Update a quota or raise if it does not exist.""" - return IMPL.quota_update(context, project_id, resource, limit) - - -def quota_destroy(context, project_id, resource): - """Destroy the quota or raise if it does not exist.""" - return IMPL.quota_destroy(context, project_id, resource) + return IMPL.quota_update(context, project_id, resource, limit, + user_id=user_id) ################### @@ -202,6 +209,11 @@ def quota_class_get(context, class_name, resource): return IMPL.quota_class_get(context, class_name, resource) +def quota_class_get_default(context): + """Retrieve all default quotas.""" + return IMPL.quota_class_get_default(context) + + def quota_class_get_all_by_name(context, class_name): """Retrieve all quotas associated with a given quota class.""" return IMPL.quota_class_get_all_by_name(context, class_name) @@ -212,29 +224,18 @@ def quota_class_update(context, class_name, resource, limit): return IMPL.quota_class_update(context, class_name, resource, limit) -def quota_class_destroy(context, class_name, resource): - """Destroy the quota class or raise if it does not exist.""" - return IMPL.quota_class_destroy(context, class_name, resource) - - -def quota_class_destroy_all_by_name(context, class_name): - """Destroy all quotas associated with a given quota class.""" - return IMPL.quota_class_destroy_all_by_name(context, class_name) - - ################### -def quota_usage_create(context, project_id, resource, in_use, reserved, - until_refresh): - """Create a quota usage for the given project and resource.""" - return IMPL.quota_usage_create(context, project_id, resource, - in_use, reserved, until_refresh) - - -def quota_usage_get(context, project_id, resource): +def quota_usage_get(context, project_id, resource, user_id=None): """Retrieve a quota usage or raise if it does not exist.""" - return IMPL.quota_usage_get(context, project_id, resource) + return IMPL.quota_usage_get(context, project_id, resource, user_id=user_id) + + +def quota_usage_get_all_by_project_and_user(context, project_id, user_id): + """Retrieve all usage associated with a given resource.""" + return IMPL.quota_usage_get_all_by_project_and_user(context, + project_id, user_id) def quota_usage_get_all_by_project(context, project_id): @@ -242,14 +243,20 @@ def quota_usage_get_all_by_project(context, project_id): return IMPL.quota_usage_get_all_by_project(context, project_id) +def quota_usage_update(context, project_id, user_id, resource, **kwargs): + """Update a quota usage or raise if it does not exist.""" + return IMPL.quota_usage_update(context, project_id, user_id, resource, + **kwargs) + + ################### -def reservation_create(context, uuid, usage, project_id, resource, delta, - expire): +def reservation_create(context, uuid, usage, project_id, user_id, resource, + delta, expire): """Create a reservation for the given project and resource.""" return IMPL.reservation_create(context, uuid, usage, project_id, - resource, delta, expire) + user_id, resource, delta, expire) def reservation_get(context, uuid): @@ -257,36 +264,35 @@ def reservation_get(context, uuid): return IMPL.reservation_get(context, uuid) -def reservation_get_all_by_project(context, project_id): - """Retrieve all reservations associated with a given project.""" - return IMPL.reservation_get_all_by_project(context, project_id) - - -def reservation_destroy(context, uuid): - """Destroy the reservation or raise if it does not exist.""" - return IMPL.reservation_destroy(context, uuid) - - ################### -def quota_reserve(context, resources, quotas, deltas, expire, - until_refresh, max_age, project_id=None): +def quota_reserve(context, resources, quotas, user_quotas, deltas, expire, + until_refresh, max_age, project_id=None, user_id=None): """Check quotas and create appropriate reservations.""" - return IMPL.quota_reserve(context, resources, quotas, deltas, expire, - until_refresh, max_age, project_id=project_id) + return IMPL.quota_reserve(context, resources, quotas, user_quotas, deltas, + expire, until_refresh, max_age, + project_id=project_id, user_id=user_id) -def reservation_commit(context, reservations, project_id=None): +def reservation_commit(context, reservations, project_id=None, user_id=None): """Commit quota reservations.""" return IMPL.reservation_commit(context, reservations, - project_id=project_id) + project_id=project_id, + user_id=user_id) -def reservation_rollback(context, reservations, project_id=None): +def reservation_rollback(context, reservations, project_id=None, user_id=None): """Roll back quota reservations.""" return IMPL.reservation_rollback(context, reservations, - project_id=project_id) + project_id=project_id, + user_id=user_id) + + +def quota_destroy_all_by_project_and_user(context, project_id, user_id): + """Destroy all quotas associated with a given project and user.""" + return IMPL.quota_destroy_all_by_project_and_user(context, + project_id, user_id) def quota_destroy_all_by_project(context, project_id): diff --git a/manila/db/sqlalchemy/api.py b/manila/db/sqlalchemy/api.py index 8d457db536..bd8c44f8a2 100644 --- a/manila/db/sqlalchemy/api.py +++ b/manila/db/sqlalchemy/api.py @@ -20,6 +20,8 @@ """Implementation of SQLAlchemy backend.""" import datetime +import functools +import time import uuid import warnings @@ -43,6 +45,9 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) +_DEFAULT_QUOTA_NAME = 'default' +PER_PROJECT_QUOTAS = [] + def is_admin_context(context): """Indicates if the request context is an administrator.""" @@ -196,6 +201,43 @@ def exact_filter(query, model, filters, legal_keys): return query +def _sync_shares(context, project_id, user_id, session): + (shares, gigs) = share_data_get_for_project(context, + project_id, + user_id, + session=session) + return {'shares': shares} + + +def _sync_snapshots(context, project_id, user_id, session): + (snapshots, gigs) = snapshot_data_get_for_project(context, + project_id, + user_id, + session=session) + return {'snapshots': snapshots} + + +def _sync_gigabytes(context, project_id, user_id, session): + (_junk, share_gigs) = share_data_get_for_project(context, + project_id, + user_id, + session=session) + if CONF.no_snapshot_gb_quota: + return {'gigabytes': share_gigs} + + (_junk, snap_gigs) = snapshot_data_get_for_project(context, + project_id, + user_id, + session=session) + return {'gigabytes': share_gigs + snap_gigs} + + +QUOTA_SYNC_FUNCTIONS = { + '_sync_shares': _sync_shares, + '_sync_snapshots': _sync_snapshots, + '_sync_gigabytes': _sync_gigabytes, +} + ################### @@ -341,6 +383,24 @@ def quota_get(context, project_id, resource, session=None): return result +@require_context +def quota_get_all_by_project_and_user(context, project_id, user_id): + authorize_project_context(context, project_id) + + user_quotas = model_query(context, models.ProjectUserQuota.resource, + models.ProjectUserQuota.hard_limit, + base_model=models.ProjectUserQuota).\ + filter_by(project_id=project_id).\ + filter_by(user_id=user_id).\ + all() + + result = {'project_id': project_id, 'user_id': user_id} + for quota in user_quotas: + result[quota.resource] = quota.hard_limit + + return result + + @require_context def quota_get_all_by_project(context, project_id): authorize_project_context(context, project_id) @@ -356,9 +416,38 @@ def quota_get_all_by_project(context, project_id): return result +@require_context +def quota_get_all(context, project_id): + authorize_project_context(context, project_id) + + result = model_query(context, models.ProjectUserQuota).\ + filter_by(project_id=project_id).\ + all() + + return result + + @require_admin_context -def quota_create(context, project_id, resource, limit): - quota_ref = models.Quota() +def quota_create(context, project_id, resource, limit, user_id=None): + per_user = user_id and resource not in PER_PROJECT_QUOTAS + + if per_user: + check = model_query(context, models.ProjectUserQuota).\ + filter_by(project_id=project_id).\ + filter_by(user_id=user_id).\ + filter_by(resource=resource).\ + all() + else: + check = model_query(context, models.Quota).\ + filter_by(project_id=project_id).\ + filter_by(resource=resource).\ + all() + if check: + raise exception.QuotaExists(project_id=project_id, resource=resource) + + quota_ref = models.ProjectUserQuota() if per_user else models.Quota() + if per_user: + quota_ref.user_id = user_id quota_ref.project_id = project_id quota_ref.resource = resource quota_ref.hard_limit = limit @@ -367,20 +456,22 @@ def quota_create(context, project_id, resource, limit): @require_admin_context -def quota_update(context, project_id, resource, limit): - session = get_session() - with session.begin(): - quota_ref = quota_get(context, project_id, resource, session=session) - quota_ref.hard_limit = limit - quota_ref.save(session=session) +def quota_update(context, project_id, resource, limit, user_id=None): + per_user = user_id and resource not in PER_PROJECT_QUOTAS + model = models.ProjectUserQuota if per_user else models.Quota + query = model_query(context, model).\ + filter_by(project_id=project_id).\ + filter_by(resource=resource) + if per_user: + query = query.filter_by(user_id=user_id) - -@require_admin_context -def quota_destroy(context, project_id, resource): - session = get_session() - with session.begin(): - quota_ref = quota_get(context, project_id, resource, session=session) - quota_ref.delete(session=session) + result = query.update({'hard_limit': limit}) + if not result: + if per_user: + raise exception.ProjectUserQuotaNotFound(project_id=project_id, + user_id=user_id) + else: + raise exception.ProjectQuotaNotFound(project_id=project_id) ################### @@ -400,6 +491,18 @@ def quota_class_get(context, class_name, resource, session=None): return result +def quota_class_get_default(context): + rows = model_query(context, models.QuotaClass, read_deleted="no").\ + filter_by(class_name=_DEFAULT_QUOTA_NAME).\ + all() + + result = {'class_name': _DEFAULT_QUOTA_NAME} + for row in rows: + result[row.resource] = row.hard_limit + + return result + + @require_context def quota_class_get_all_by_name(context, class_name): authorize_quota_class_context(context, class_name) @@ -427,46 +530,30 @@ def quota_class_create(context, class_name, resource, limit): @require_admin_context def quota_class_update(context, class_name, resource, limit): - session = get_session() - with session.begin(): - quota_class_ref = quota_class_get(context, class_name, resource, - session=session) - quota_class_ref.hard_limit = limit - quota_class_ref.save(session=session) + result = model_query(context, models.QuotaClass, read_deleted="no").\ + filter_by(class_name=class_name).\ + filter_by(resource=resource).\ + update({'hard_limit': limit}) - -@require_admin_context -def quota_class_destroy(context, class_name, resource): - session = get_session() - with session.begin(): - quota_class_ref = quota_class_get(context, class_name, resource, - session=session) - quota_class_ref.delete(session=session) - - -@require_admin_context -def quota_class_destroy_all_by_name(context, class_name): - session = get_session() - with session.begin(): - quota_classes = model_query(context, models.QuotaClass, - session=session, read_deleted="no").\ - filter_by(class_name=class_name).\ - all() - - for quota_class_ref in quota_classes: - quota_class_ref.delete(session=session) + if not result: + raise exception.QuotaClassNotFound(class_name=class_name) ################### @require_context -def quota_usage_get(context, project_id, resource, session=None): - result = model_query(context, models.QuotaUsage, session=session, - read_deleted="no").\ - filter_by(project_id=project_id).\ - filter_by(resource=resource).\ - first() +def quota_usage_get(context, project_id, resource, user_id=None): + query = model_query(context, models.QuotaUsage, read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(resource=resource) + if user_id: + if resource not in PER_PROJECT_QUOTAS: + result = query.filter_by(user_id=user_id).first() + else: + result = query.filter_by(user_id=None).first() + else: + result = query.first() if not result: raise exception.QuotaUsageNotFound(project_id=project_id) @@ -474,35 +561,74 @@ def quota_usage_get(context, project_id, resource, session=None): return result -@require_context -def quota_usage_get_all_by_project(context, project_id): +def _quota_usage_get_all(context, project_id, user_id=None): authorize_project_context(context, project_id) - - rows = model_query(context, models.QuotaUsage, read_deleted="no").\ - filter_by(project_id=project_id).\ - all() - + query = model_query(context, models.QuotaUsage, read_deleted="no").\ + filter_by(project_id=project_id) result = {'project_id': project_id} + if user_id: + query = query.filter(or_(models.QuotaUsage.user_id == user_id, + models.QuotaUsage.user_id == None)) + result['user_id'] = user_id + + rows = query.all() for row in rows: - result[row.resource] = dict(in_use=row.in_use, reserved=row.reserved) + if row.resource in result: + result[row.resource]['in_use'] += row.in_use + result[row.resource]['reserved'] += row.reserved + else: + result[row.resource] = dict(in_use=row.in_use, + reserved=row.reserved) return result -@require_admin_context -def quota_usage_create(context, project_id, resource, in_use, reserved, - until_refresh, session=None): +@require_context +def quota_usage_get_all_by_project(context, project_id): + return _quota_usage_get_all(context, project_id) + + +@require_context +def quota_usage_get_all_by_project_and_user(context, project_id, user_id): + return _quota_usage_get_all(context, project_id, user_id=user_id) + + +def _quota_usage_create(context, project_id, user_id, resource, in_use, + reserved, until_refresh, session=None): quota_usage_ref = models.QuotaUsage() quota_usage_ref.project_id = project_id + quota_usage_ref.user_id = user_id quota_usage_ref.resource = resource quota_usage_ref.in_use = in_use quota_usage_ref.reserved = reserved quota_usage_ref.until_refresh = until_refresh + # updated_at is needed for judgement of max_age + quota_usage_ref.updated_at = timeutils.utcnow() + quota_usage_ref.save(session=session) return quota_usage_ref +@require_admin_context +def quota_usage_update(context, project_id, user_id, resource, **kwargs): + updates = {} + + for key in ['in_use', 'reserved', 'until_refresh']: + if key in kwargs: + updates[key] = kwargs[key] + + result = model_query(context, models.QuotaUsage, read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(resource=resource).\ + filter(or_(models.QuotaUsage.user_id == user_id, + models.QuotaUsage.user_id == None)).\ + update(updates) + + if not result: + raise exception.QuotaUsageNotFound(project_id=project_id) + + ################### @@ -518,28 +644,20 @@ def reservation_get(context, uuid, session=None): return result -@require_context -def reservation_get_all_by_project(context, project_id): - authorize_project_context(context, project_id) - - rows = model_query(context, models.Reservation, read_deleted="no").\ - filter_by(project_id=project_id).all() - - result = {'project_id': project_id} - for row in rows: - result.setdefault(row.resource, {}) - result[row.resource][row.uuid] = row.delta - - return result - - @require_admin_context -def reservation_create(context, uuid, usage, project_id, resource, delta, - expire, session=None): +def reservation_create(context, uuid, usage, project_id, user_id, resource, + delta, expire): + return _reservation_create(context, uuid, usage, project_id, user_id, + resource, delta, expire) + + +def _reservation_create(context, uuid, usage, project_id, user_id, resource, + delta, expire, session=None): reservation_ref = models.Reservation() reservation_ref.uuid = uuid reservation_ref.usage_id = usage['id'] reservation_ref.project_id = project_id + reservation_ref.user_id = user_id reservation_ref.resource = resource reservation_ref.delta = delta reservation_ref.expire = expire @@ -547,14 +665,6 @@ def reservation_create(context, uuid, usage, project_id, resource, delta, return reservation_ref -@require_admin_context -def reservation_destroy(context, uuid): - session = get_session() - with session.begin(): - reservation_ref = reservation_get(context, uuid, session=session) - reservation_ref.delete(session=session) - - ################### @@ -563,28 +673,58 @@ def reservation_destroy(context, uuid): # code always acquires the lock on quota_usages before acquiring the lock # on reservations. -def _get_quota_usages(context, session, project_id): +def _get_user_quota_usages(context, session, project_id, user_id): # Broken out for testability rows = model_query(context, models.QuotaUsage, read_deleted="no", session=session).\ - filter_by(project_id=project_id).\ - with_lockmode('update').\ - all() + filter_by(project_id=project_id).\ + filter(or_(models.QuotaUsage.user_id == user_id, + models.QuotaUsage.user_id == None)).\ + with_lockmode('update').\ + all() return dict((row.resource, row) for row in rows) +def _get_project_quota_usages(context, session, project_id): + rows = model_query(context, models.QuotaUsage, + read_deleted="no", + session=session).\ + filter_by(project_id=project_id).\ + with_lockmode('update').\ + all() + result = dict() + # Get the total count of in_use,reserved + for row in rows: + if row.resource in result: + result[row.resource]['in_use'] += row.in_use + result[row.resource]['reserved'] += row.reserved + result[row.resource]['total'] += (row.in_use + row.reserved) + else: + result[row.resource] = dict(in_use=row.in_use, + reserved=row.reserved, + total=row.in_use + row.reserved) + return result + + @require_context -def quota_reserve(context, resources, quotas, deltas, expire, - until_refresh, max_age, project_id=None): +def quota_reserve(context, resources, project_quotas, user_quotas, deltas, + expire, until_refresh, max_age, project_id=None, + user_id=None): elevated = context.elevated() session = get_session() with session.begin(): + if project_id is None: project_id = context.project_id + if user_id is None: + user_id = context.user_id # Get the current usages - usages = _get_quota_usages(context, session, project_id) + user_usages = _get_user_quota_usages(context, session, + project_id, user_id) + project_usages = _get_project_quota_usages(context, session, + project_id) # Handle usage refresh work = set(deltas.keys()) @@ -593,45 +733,81 @@ def quota_reserve(context, resources, quotas, deltas, expire, # Do we need to refresh the usage? refresh = False - if resource not in usages: - usages[resource] = quota_usage_create(elevated, + if ((resource not in PER_PROJECT_QUOTAS) and + (resource not in user_usages)): + user_usages[resource] = _quota_usage_create(elevated, project_id, + user_id, resource, 0, 0, until_refresh or None, session=session) refresh = True - elif usages[resource].in_use < 0: + elif ((resource in PER_PROJECT_QUOTAS) and + (resource not in user_usages)): + user_usages[resource] = _quota_usage_create(elevated, + project_id, + None, + resource, + 0, 0, + until_refresh or None, + session=session) + refresh = True + elif user_usages[resource].in_use < 0: # Negative in_use count indicates a desync, so try to # heal from that... refresh = True - elif usages[resource].until_refresh is not None: - usages[resource].until_refresh -= 1 - if usages[resource].until_refresh <= 0: + elif user_usages[resource].until_refresh is not None: + user_usages[resource].until_refresh -= 1 + if user_usages[resource].until_refresh <= 0: refresh = True - elif max_age and (usages[resource].updated_at - + elif max_age and (user_usages[resource].updated_at - timeutils.utcnow()).seconds >= max_age: refresh = True # OK, refresh the usage if refresh: # Grab the sync routine - sync = resources[resource].sync + sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync] - updates = sync(elevated, project_id, session) + updates = sync(elevated, project_id, user_id, session) for res, in_use in updates.items(): # Make sure we have a destination for the usage! - if res not in usages: - usages[res] = quota_usage_create(elevated, + if ((res not in PER_PROJECT_QUOTAS) and + (res not in user_usages)): + user_usages[res] = _quota_usage_create(elevated, project_id, + user_id, + res, + 0, 0, + until_refresh or None, + session=session) + if ((res in PER_PROJECT_QUOTAS) and + (res not in user_usages)): + user_usages[res] = _quota_usage_create(elevated, + project_id, + None, res, 0, 0, until_refresh or None, session=session) + if user_usages[res].in_use != in_use: + LOG.debug(_('quota_usages out of sync, updating. ' + 'project_id: %(project_id)s, ' + 'user_id: %(user_id)s, ' + 'resource: %(res)s, ' + 'tracked usage: %(tracked_use)s, ' + 'actual usage: %(in_use)s'), + {'project_id': project_id, + 'user_id': user_id, + 'res': res, + 'tracked_use': user_usages[res].in_use, + 'in_use': in_use}) + # Update the usage - usages[res].in_use = in_use - usages[res].until_refresh = until_refresh or None + user_usages[res].in_use = in_use + user_usages[res].until_refresh = until_refresh or None # Because more than one resource may be refreshed # by the call to the sync routine, and we don't @@ -646,18 +822,24 @@ def quota_reserve(context, resources, quotas, deltas, expire, # a best-effort mechanism. # Check for deltas that would go negative - unders = [resource for resource, delta in deltas.items() + unders = [res for res, delta in deltas.items() if delta < 0 and - delta + usages[resource].in_use < 0] + delta + user_usages[res].in_use < 0] # Now, let's check the quotas # NOTE(Vek): We're only concerned about positive increments. # If a project has gone over quota, we want them to # be able to reduce their usage without any # problems. - overs = [resource for resource, delta in deltas.items() - if quotas[resource] >= 0 and delta >= 0 and - quotas[resource] < delta + usages[resource].total] + for key, value in user_usages.items(): + if key not in project_usages: + project_usages[key] = value + overs = [res for res, delta in deltas.items() + if user_quotas[res] >= 0 and delta >= 0 and + (project_quotas[res] < delta + + project_usages[res]['total'] or + user_quotas[res] < delta + + user_usages[res].total)] # NOTE(Vek): The quota check needs to be in the transaction, # but the transaction doesn't fail just because @@ -669,12 +851,13 @@ def quota_reserve(context, resources, quotas, deltas, expire, # Create the reservations if not overs: reservations = [] - for resource, delta in deltas.items(): - reservation = reservation_create(elevated, + for res, delta in deltas.items(): + reservation = _reservation_create(elevated, str(uuid.uuid4()), - usages[resource], + user_usages[res], project_id, - resource, delta, expire, + user_id, + res, delta, expire, session=session) reservations.append(reservation.uuid) @@ -691,98 +874,141 @@ def quota_reserve(context, resources, quotas, deltas, expire, # To prevent this, we only update the # reserved value if the delta is positive. if delta > 0: - usages[resource].reserved += delta + user_usages[res].reserved += delta # Apply updates to the usages table - for usage_ref in usages.values(): - usage_ref.save(session=session) + for usage_ref in user_usages.values(): + session.add(usage_ref) if unders: LOG.warning(_("Change will make usage less than 0 for the following " - "resources: %(unders)s") % locals()) + "resources: %s"), unders) if overs: + if project_quotas == user_quotas: + usages = project_usages + else: + usages = user_usages usages = dict((k, dict(in_use=v['in_use'], reserved=v['reserved'])) for k, v in usages.items()) - raise exception.OverQuota(overs=sorted(overs), quotas=quotas, + raise exception.OverQuota(overs=sorted(overs), quotas=user_quotas, usages=usages) return reservations -def _quota_reservations(session, context, reservations): +def _quota_reservations_query(session, context, reservations): """Return the relevant reservations.""" # Get the listed reservations return model_query(context, models.Reservation, read_deleted="no", session=session).\ - filter(models.Reservation.uuid.in_(reservations)).\ - with_lockmode('update').\ - all() + filter(models.Reservation.uuid.in_(reservations)).\ + with_lockmode('update') @require_context -def reservation_commit(context, reservations, project_id=None): +def reservation_commit(context, reservations, project_id=None, user_id=None): session = get_session() with session.begin(): - usages = _get_quota_usages(context, session, project_id) - - for reservation in _quota_reservations(session, context, reservations): + usages = _get_user_quota_usages(context, session, project_id, user_id) + reservation_query = _quota_reservations_query(session, context, + reservations) + for reservation in reservation_query.all(): usage = usages[reservation.resource] if reservation.delta >= 0: usage.reserved -= reservation.delta usage.in_use += reservation.delta - - reservation.delete(session=session) - - for usage in usages.values(): - usage.save(session=session) + reservation_query.update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) @require_context -def reservation_rollback(context, reservations, project_id=None): +def reservation_rollback(context, reservations, project_id=None, user_id=None): session = get_session() with session.begin(): - usages = _get_quota_usages(context, session, project_id) - - for reservation in _quota_reservations(session, context, reservations): + usages = _get_user_quota_usages(context, session, project_id, user_id) + reservation_query = _quota_reservations_query(session, context, + reservations) + for reservation in reservation_query.all(): usage = usages[reservation.resource] if reservation.delta >= 0: usage.reserved -= reservation.delta + reservation_query.update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) - reservation.delete(session=session) - for usage in usages.values(): - usage.save(session=session) +@require_admin_context +def quota_destroy_all_by_project_and_user(context, project_id, user_id): + session = get_session() + with session.begin(): + model_query(context, models.ProjectUserQuota, session=session, + read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(user_id=user_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) + + model_query(context, models.QuotaUsage, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(user_id=user_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) + + model_query(context, models.Reservation, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + filter_by(user_id=user_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) @require_admin_context def quota_destroy_all_by_project(context, project_id): session = get_session() with session.begin(): - quotas = model_query(context, models.Quota, session=session, - read_deleted="no").\ - filter_by(project_id=project_id).\ - all() + model_query(context, models.Quota, session=session, + read_deleted="no").\ + filter_by(project_id=project_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) - for quota_ref in quotas: - quota_ref.delete(session=session) + model_query(context, models.ProjectUserQuota, session=session, + read_deleted="no").\ + filter_by(project_id=project_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) - quota_usages = model_query(context, models.QuotaUsage, - session=session, read_deleted="no").\ - filter_by(project_id=project_id).\ - all() + model_query(context, models.QuotaUsage, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) - for quota_usage_ref in quota_usages: - quota_usage_ref.delete(session=session) - - reservations = model_query(context, models.Reservation, - session=session, read_deleted="no").\ - filter_by(project_id=project_id).\ - all() - - for reservation_ref in reservations: - reservation_ref.delete(session=session) + model_query(context, models.Reservation, + session=session, read_deleted="no").\ + filter_by(project_id=project_id).\ + update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) @require_admin_context @@ -790,18 +1016,19 @@ def reservation_expire(context): session = get_session() with session.begin(): current_time = timeutils.utcnow() - results = model_query(context, models.Reservation, session=session, - read_deleted="no").\ - filter(models.Reservation.expire < current_time).\ - all() + reservation_query = model_query(context, models.Reservation, + session=session, read_deleted="no").\ + filter(models.Reservation.expire < current_time) - if results: - for reservation in results: - if reservation.delta >= 0: - reservation.usage.reserved -= reservation.delta - reservation.usage.save(session=session) + for reservation in reservation_query.join(models.QuotaUsage).all(): + if reservation.delta >= 0: + reservation.usage.reserved -= reservation.delta + session.add(reservation.usage) - reservation.delete(session=session) + reservation_query.update({'deleted': True, + 'deleted_at': timeutils.utcnow(), + 'updated_at': literal_column('updated_at')}, + synchronize_session=False) ################ @@ -827,15 +1054,17 @@ def share_create(context, values): @require_admin_context -def share_data_get_for_project(context, project_id, session=None): +def share_data_get_for_project(context, project_id, user_id, session=None): query = model_query(context, func.count(models.Share.id), func.sum(models.Share.size), read_deleted="no", session=session).\ filter_by(project_id=project_id) - - result = query.first() + if user_id: + result = query.filter_by(user_id=user_id).first() + else: + result = query.first() return (result[0] or 0, result[1] or 0) @@ -971,15 +1200,17 @@ def share_snapshot_create(context, values): @require_admin_context -def snapshot_data_get_for_project(context, project_id, session=None): +def snapshot_data_get_for_project(context, project_id, user_id, session=None): query = model_query(context, func.count(models.ShareSnapshot.id), func.sum(models.ShareSnapshot.size), read_deleted="no", session=session).\ filter_by(project_id=project_id) - - result = query.first() + if user_id: + result = query.filter_by(user_id=user_id).first() + else: + result = query.first() return (result[0] or 0, result[1] or 0) diff --git a/manila/db/sqlalchemy/migrate_repo/versions/005_make_user_quotas_key_and_value.py b/manila/db/sqlalchemy/migrate_repo/versions/005_make_user_quotas_key_and_value.py new file mode 100644 index 0000000000..9d656c1b1c --- /dev/null +++ b/manila/db/sqlalchemy/migrate_repo/versions/005_make_user_quotas_key_and_value.py @@ -0,0 +1,88 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, DateTime, Integer +from sqlalchemy import Index, UniqueConstraint, MetaData, String, Table + +from manila.db.sqlalchemy import api as db +from manila.openstack.common.gettextutils import _ +from manila.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta = MetaData() + meta.bind = migrate_engine + + # Add 'user_id' column to quota_usages table. + quota_usages = Table('quota_usages', meta, autoload=True) + user_id = Column('user_id', + String(length=255)) + quota_usages.create_column(user_id) + + # Add 'user_id' column to reservations table. + reservations = Table('reservations', meta, autoload=True) + user_id = Column('user_id', + String(length=255)) + reservations.create_column(user_id) + + project_user_quotas = Table('project_user_quotas', meta, + Column('id', Integer, primary_key=True, + nullable=False), + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('deleted', Integer), + Column('user_id', + String(length=255), + nullable=False), + Column('project_id', + String(length=255), + nullable=False), + Column('resource', + String(length=25), + nullable=False), + Column('hard_limit', Integer, nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + try: + project_user_quotas.create() + except Exception: + LOG.exception("Exception while creating table 'project_user_quotas'") + meta.drop_all(tables=[project_user_quotas]) + raise + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + quota_usages = Table('quota_usages', meta, autoload=True) + reservations = Table('reservations', meta, autoload=True) + + quota_usages.drop_column('user_id') + reservations.drop_column('user_id') + + project_user_quotas = Table('project_user_quotas', meta, autoload=True) + try: + project_user_quotas.drop() + except Exception: + LOG.error(_("project_user_quotas table not dropped")) + raise diff --git a/manila/db/sqlalchemy/models.py b/manila/db/sqlalchemy/models.py index 9f824d86f3..64cfc75dcc 100644 --- a/manila/db/sqlalchemy/models.py +++ b/manila/db/sqlalchemy/models.py @@ -141,6 +141,19 @@ class Quota(BASE, ManilaBase): hard_limit = Column(Integer, nullable=True) +class ProjectUserQuota(BASE, ManilaBase): + """Represents a single quota override for a user with in a project.""" + + __tablename__ = 'project_user_quotas' + id = Column(Integer, primary_key=True, nullable=False) + + project_id = Column(String(255), nullable=False) + user_id = Column(String(255), nullable=False) + + resource = Column(String(255), nullable=False) + hard_limit = Column(Integer) + + class QuotaClass(BASE, ManilaBase): """Represents a single quota override for a quota class. @@ -165,6 +178,7 @@ class QuotaUsage(BASE, ManilaBase): id = Column(Integer, primary_key=True) project_id = Column(String(255), index=True) + user_id = Column(String(255)) resource = Column(String(255)) in_use = Column(Integer) @@ -187,11 +201,18 @@ class Reservation(BASE, ManilaBase): usage_id = Column(Integer, ForeignKey('quota_usages.id'), nullable=False) project_id = Column(String(255), index=True) + user_id = Column(String(255)) resource = Column(String(255)) delta = Column(Integer) expire = Column(DateTime, nullable=False) +# usage = relationship( +# "QuotaUsage", +# foreign_keys=usage_id, +# primaryjoin='and_(Reservation.usage_id == QuotaUsage.id,' +# 'QuotaUsage.deleted == 0)') + class Migration(BASE, ManilaBase): """Represents a running host-to-host migration.""" diff --git a/manila/exception.py b/manila/exception.py index 95ac8f2cba..b3b535aeb6 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -242,36 +242,46 @@ class InvalidReservationExpiration(Invalid): class InvalidQuotaValue(Invalid): - message = _("Change would make usage less than 0 for the following " + msg_fmt = _("Change would make usage less than 0 for the following " "resources: %(unders)s") class QuotaNotFound(NotFound): - message = _("Quota could not be found") + msg_fmt = _("Quota could not be found") + + +class QuotaExists(ManilaException): + msg_fmt = _("Quota exists for project %(project_id)s, " + "resource %(resource)s") class QuotaResourceUnknown(QuotaNotFound): - message = _("Unknown quota resources %(unknown)s.") + msg_fmt = _("Unknown quota resources %(unknown)s.") + + +class ProjectUserQuotaNotFound(QuotaNotFound): + msg_fmt = _("Quota for user %(user_id)s in project %(project_id)s " + "could not be found.") class ProjectQuotaNotFound(QuotaNotFound): - message = _("Quota for project %(project_id)s could not be found.") + msg_fmt = _("Quota for project %(project_id)s could not be found.") class QuotaClassNotFound(QuotaNotFound): - message = _("Quota class %(class_name)s could not be found.") + msg_fmt = _("Quota class %(class_name)s could not be found.") class QuotaUsageNotFound(QuotaNotFound): - message = _("Quota usage for project %(project_id)s could not be found.") + msg_fmt = _("Quota usage for project %(project_id)s could not be found.") class ReservationNotFound(QuotaNotFound): - message = _("Quota reservation %(uuid)s could not be found.") + msg_fmt = _("Quota reservation %(uuid)s could not be found.") class OverQuota(ManilaException): - message = _("Quota exceeded for resources: %(overs)s") + msg_fmt = _("Quota exceeded for resources: %(overs)s") class MigrationNotFound(NotFound): diff --git a/manila/quota.py b/manila/quota.py index 51ab16adf4..cad5890e16 100644 --- a/manila/quota.py +++ b/manila/quota.py @@ -64,6 +64,10 @@ class DbQuotaDriver(object): quota information. The default driver utilizes the local database. """ + def get_by_project_and_user(self, context, project_id, user_id, resource): + """Get a specific quota by project and user.""" + + return db.quota_get(context, project_id, user_id, resource) def get_by_project(self, context, project_id, resource): """Get a specific quota by project.""" @@ -83,8 +87,10 @@ class DbQuotaDriver(object): """ quotas = {} + default_quotas = db.quota_class_get_default(context) for resource in resources.values(): - quotas[resource.name] = resource.default + quotas[resource.name] = default_quotas.get(resource.name, + resource.default) return quotas @@ -112,9 +118,57 @@ class DbQuotaDriver(object): return quotas + def _process_quotas(self, context, resources, project_id, quotas, + quota_class=None, defaults=True, usages=None, + remains=False): + modified_quotas = {} + # Get the quotas for the appropriate class. If the project ID + # matches the one in the context, we use the quota_class from + # the context, otherwise, we use the provided quota_class (if + # any) + if project_id == context.project_id: + quota_class = context.quota_class + if quota_class: + class_quotas = db.quota_class_get_all_by_name(context, quota_class) + else: + class_quotas = {} + + default_quotas = self.get_defaults(context, resources) + + for resource in resources.values(): + # Omit default/quota class values + if not defaults and resource.name not in quotas: + continue + + limit = quotas.get(resource.name, class_quotas.get( + resource.name, default_quotas[resource.name])) + modified_quotas[resource.name] = dict(limit=limit) + + # Include usages if desired. This is optional because one + # internal consumer of this interface wants to access the + # usages directly from inside a transaction. + if usages: + usage = usages.get(resource.name, {}) + modified_quotas[resource.name].update( + in_use=usage.get('in_use', 0), + reserved=usage.get('reserved', 0), + ) + # Initialize remains quotas. + if remains: + modified_quotas[resource.name].update(remains=limit) + + if remains: + all_quotas = db.quota_get_all(context, project_id) + for quota in all_quotas: + if quota.resource in modified_quotas: + modified_quotas[quota.resource]['remains'] -= \ + quota.hard_limit + + return modified_quotas + def get_project_quotas(self, context, resources, project_id, quota_class=None, defaults=True, - usages=True): + usages=True, remains=False): """ Given a list of resources, retrieve the quotas for the given project. @@ -133,47 +187,94 @@ class DbQuotaDriver(object): specific value for the resource. :param usages: If True, the current in_use and reserved counts will also be returned. + :param remains: If True, the current remains of the project will + will be returned. """ - - quotas = {} project_quotas = db.quota_get_all_by_project(context, project_id) + project_usages = None if usages: project_usages = db.quota_usage_get_all_by_project(context, project_id) + return self._process_quotas(context, resources, project_id, + project_quotas, quota_class, + defaults=defaults, usages=project_usages, + remains=remains) - # Get the quotas for the appropriate class. If the project ID - # matches the one in the context, we use the quota_class from - # the context, otherwise, we use the provided quota_class (if - # any) - if project_id == context.project_id: - quota_class = context.quota_class - if quota_class: - class_quotas = db.quota_class_get_all_by_name(context, quota_class) + def get_user_quotas(self, context, resources, project_id, user_id, + quota_class=None, defaults=True, + usages=True): + """ + Given a list of resources, retrieve the quotas for the given + user and project. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param project_id: The ID of the project to return quotas for. + :param user_id: The ID of the user to return quotas for. + :param quota_class: If project_id != context.project_id, the + quota class cannot be determined. This + parameter allows it to be specified. It + will be ignored if project_id == + context.project_id. + :param defaults: If True, the quota class value (or the + default value, if there is no value from the + quota class) will be reported if there is no + specific value for the resource. + :param usages: If True, the current in_use and reserved counts + will also be returned. + """ + user_quotas = db.quota_get_all_by_project_and_user(context, + project_id, user_id) + # Use the project quota for default user quota. + proj_quotas = db.quota_get_all_by_project(context, project_id) + for key, value in proj_quotas.iteritems(): + if key not in user_quotas.keys(): + user_quotas[key] = value + user_usages = None + if usages: + user_usages = db.quota_usage_get_all_by_project_and_user(context, + project_id, + user_id) + return self._process_quotas(context, resources, project_id, + user_quotas, quota_class, + defaults=defaults, usages=user_usages) + + def get_settable_quotas(self, context, resources, project_id, + user_id=None): + """ + Given a list of resources, retrieve the range of settable quotas for + the given user or project. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param project_id: The ID of the project to return quotas for. + :param user_id: The ID of the user to return quotas for. + """ + settable_quotas = {} + project_quotas = self.get_project_quotas(context, resources, + project_id, remains=True) + if user_id: + user_quotas = self.get_user_quotas(context, resources, + project_id, user_id) + setted_quotas = db.quota_get_all_by_project_and_user(context, + project_id, + user_id) + for key, value in user_quotas.items(): + maximum = project_quotas[key]['remains'] +\ + setted_quotas.get(key, 0) + settable_quotas[key] = dict( + minimum=value['in_use'] + value['reserved'], + maximum=maximum + ) else: - class_quotas = {} + for key, value in project_quotas.items(): + minimum = max(int(value['limit'] - value['remains']), + int(value['in_use'] + value['reserved'])) + settable_quotas[key] = dict(minimum=minimum, maximum=-1) + return settable_quotas - for resource in resources.values(): - # Omit default/quota class values - if not defaults and resource.name not in project_quotas: - continue - - quotas[resource.name] = dict( - limit=project_quotas.get(resource.name, - class_quotas.get(resource.name, - resource.default)), ) - - # Include usages if desired. This is optional because one - # internal consumer of this interface wants to access the - # usages directly from inside a transaction. - if usages: - usage = project_usages.get(resource.name, {}) - quotas[resource.name].update( - in_use=usage.get('in_use', 0), - reserved=usage.get('reserved', 0), ) - - return quotas - - def _get_quotas(self, context, resources, keys, has_sync, project_id=None): + def _get_quotas(self, context, resources, keys, has_sync, project_id=None, + user_id=None): """ A helper method which retrieves the quotas for the specific resources identified by keys, and which apply to the current @@ -189,6 +290,9 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ # Filter resources @@ -205,14 +309,22 @@ class DbQuotaDriver(object): unknown = desired - set(sub_resources.keys()) raise exception.QuotaResourceUnknown(unknown=sorted(unknown)) - # Grab and return the quotas (without usages) - quotas = self.get_project_quotas(context, sub_resources, - project_id, - context.quota_class, usages=False) + if user_id: + # Grab and return the quotas (without usages) + quotas = self.get_user_quotas(context, sub_resources, + project_id, user_id, + context.quota_class, usages=False) + else: + # Grab and return the quotas (without usages) + quotas = self.get_project_quotas(context, sub_resources, + project_id, + context.quota_class, + usages=False) return dict((k, v['limit']) for k, v in quotas.items()) - def limit_check(self, context, resources, values, project_id=None): + def limit_check(self, context, resources, values, project_id=None, + user_id=None): """Check simple quota limits. For limits--those quotas for which there is no usage @@ -235,6 +347,9 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ # Ensure no value is less than zero @@ -245,20 +360,28 @@ class DbQuotaDriver(object): # If project_id is None, then we use the project_id in context if project_id is None: project_id = context.project_id + # If user id is None, then we use the user_id in context + if user_id is None: + user_id = context.user_id # Get the applicable quotas quotas = self._get_quotas(context, resources, values.keys(), has_sync=False, project_id=project_id) + user_quotas = self._get_quotas(context, resources, values.keys(), + has_sync=False, project_id=project_id, + user_id=user_id) + # Check the quotas and construct a list of the resources that # would be put over limit by the desired values overs = [key for key, val in values.items() - if quotas[key] >= 0 and quotas[key] < val] + if (quotas[key] >= 0 and quotas[key] < val) or + (user_quotas[key] >= 0 and user_quotas[key] < val)] if overs: raise exception.OverQuota(overs=sorted(overs), quotas=quotas, usages={}) def reserve(self, context, resources, deltas, expire=None, - project_id=None): + project_id=None, user_id=None): """Check quotas and reserve resources. For counting quotas--those quotas for which there is a usage @@ -291,6 +414,9 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ # Set up the reservation expiration @@ -306,6 +432,9 @@ class DbQuotaDriver(object): # If project_id is None, then we use the project_id in context if project_id is None: project_id = context.project_id + # If user_id is None, then we use the project_id in context + if user_id is None: + user_id = context.user_id # Get the applicable quotas. # NOTE(Vek): We're not worried about races at this point. @@ -313,17 +442,21 @@ class DbQuotaDriver(object): # quotas, but that's a pretty rare thing. quotas = self._get_quotas(context, resources, deltas.keys(), has_sync=True, project_id=project_id) + user_quotas = self._get_quotas(context, resources, deltas.keys(), + has_sync=True, project_id=project_id, + user_id=user_id) # NOTE(Vek): Most of the work here has to be done in the DB # API, because we have to do it in a transaction, # which means access to the session. Since the # session isn't available outside the DBAPI, we # have to do the work there. - return db.quota_reserve(context, resources, quotas, deltas, expire, + return db.quota_reserve(context, resources, quotas, user_quotas, + deltas, expire, CONF.until_refresh, CONF.max_age, - project_id=project_id) + project_id=project_id, user_id=user_id) - def commit(self, context, reservations, project_id=None): + def commit(self, context, reservations, project_id=None, user_id=None): """Commit reservations. :param context: The request context, for access checks. @@ -332,14 +465,21 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ # If project_id is None, then we use the project_id in context if project_id is None: project_id = context.project_id + # If user_id is None, then we use the user_id in context + if user_id is None: + user_id = context.user_id - db.reservation_commit(context, reservations, project_id=project_id) + db.reservation_commit(context, reservations, project_id=project_id, + user_id=user_id) - def rollback(self, context, reservations, project_id=None): + def rollback(self, context, reservations, project_id=None, user_id=None): """Roll back reservations. :param context: The request context, for access checks. @@ -348,12 +488,49 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ # If project_id is None, then we use the project_id in context if project_id is None: project_id = context.project_id + # If user_id is None, then we use the user_id in context + if user_id is None: + user_id = context.user_id - db.reservation_rollback(context, reservations, project_id=project_id) + db.reservation_rollback(context, reservations, project_id=project_id, + user_id=user_id) + + def usage_reset(self, context, resources): + """ + Reset the usage records for a particular user on a list of + resources. This will force that user's usage records to be + refreshed the next time a reservation is made. + + Note: this does not affect the currently outstanding + reservations the user has; those reservations must be + committed or rolled back (or expired). + + :param context: The request context, for access checks. + :param resources: A list of the resource names for which the + usage must be reset. + """ + + # We need an elevated context for the calls to + # quota_usage_update() + elevated = context.elevated() + + for resource in resources: + try: + # Reset the usage to -1, which will force it to be + # refreshed + db.quota_usage_update(elevated, context.project_id, + context.user_id, + resource, in_use=-1) + except exception.QuotaUsageNotFound: + # That means it'll be refreshed anyway + pass def destroy_all_by_project(self, context, project_id): """ @@ -366,6 +543,18 @@ class DbQuotaDriver(object): db.quota_destroy_all_by_project(context, project_id) + def destroy_all_by_project_and_user(self, context, project_id, user_id): + """ + Destroy all quotas, usages, and reservations associated with a + project and user. + + :param context: The request context, for access checks. + :param project_id: The ID of the project being deleted. + :param user_id: The ID of the user being deleted. + """ + + db.quota_destroy_all_by_project_and_user(context, project_id, user_id) + def expire(self, context): """Expire reservations. @@ -535,15 +724,20 @@ class QuotaEngine(object): def __init__(self, quota_driver_class=None): """Initialize a Quota object.""" - - if not quota_driver_class: - quota_driver_class = CONF.quota_driver - - if isinstance(quota_driver_class, basestring): - quota_driver_class = importutils.import_object(quota_driver_class) - self._resources = {} - self._driver = quota_driver_class + self._driver_cls = quota_driver_class + self.__driver = None + + @property + def _driver(self): + if self.__driver: + return self.__driver + if not self._driver_cls: + self._driver_cls = CONF.quota_driver + if isinstance(self._driver_cls, basestring): + self._driver_cls = importutils.import_object(self._driver_cls) + self.__driver = self._driver_cls + return self.__driver def __contains__(self, resource): return resource in self._resources @@ -559,6 +753,12 @@ class QuotaEngine(object): for resource in resources: self.register_resource(resource) + def get_by_project_and_user(self, context, project_id, user_id, resource): + """Get a specific quota by project and user.""" + + return self._driver.get_by_project_and_user(context, project_id, + user_id, resource) + def get_by_project(self, context, project_id, resource): """Get a specific quota by project.""" @@ -591,8 +791,32 @@ class QuotaEngine(object): return self._driver.get_class_quotas(context, self._resources, quota_class, defaults=defaults) + def get_user_quotas(self, context, project_id, user_id, quota_class=None, + defaults=True, usages=True): + """Retrieve the quotas for the given user and project. + + :param context: The request context, for access checks. + :param project_id: The ID of the project to return quotas for. + :param user_id: The ID of the user to return quotas for. + :param quota_class: If project_id != context.project_id, the + quota class cannot be determined. This + parameter allows it to be specified. + :param defaults: If True, the quota class value (or the + default value, if there is no value from the + quota class) will be reported if there is no + specific value for the resource. + :param usages: If True, the current in_use and reserved counts + will also be returned. + """ + + return self._driver.get_user_quotas(context, self._resources, + project_id, user_id, + quota_class=quota_class, + defaults=defaults, + usages=usages) + def get_project_quotas(self, context, project_id, quota_class=None, - defaults=True, usages=True): + defaults=True, usages=True, remains=False): """Retrieve the quotas for the given project. :param context: The request context, for access checks. @@ -606,13 +830,31 @@ class QuotaEngine(object): specific value for the resource. :param usages: If True, the current in_use and reserved counts will also be returned. + :param remains: If True, the current remains of the project will + will be returned. """ return self._driver.get_project_quotas(context, self._resources, - project_id, - quota_class=quota_class, - defaults=defaults, - usages=usages) + project_id, + quota_class=quota_class, + defaults=defaults, + usages=usages, + remains=remains) + + def get_settable_quotas(self, context, project_id, user_id=None): + """ + Given a list of resources, retrieve the range of settable quotas for + the given user or project. + + :param context: The request context, for access checks. + :param resources: A dictionary of the registered resources. + :param project_id: The ID of the project to return quotas for. + :param user_id: The ID of the user to return quotas for. + """ + + return self._driver.get_settable_quotas(context, self._resources, + project_id, + user_id=user_id) def count(self, context, resource, *args, **kwargs): """Count a resource. @@ -633,7 +875,7 @@ class QuotaEngine(object): return res.count(context, *args, **kwargs) - def limit_check(self, context, project_id=None, **values): + def limit_check(self, context, project_id=None, user_id=None, **values): """Check simple quota limits. For limits--those quotas for which there is no usage @@ -656,12 +898,16 @@ class QuotaEngine(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. + :param user_id: Specify the user_id if current context + is admin and admin wants to impact on + common user. """ return self._driver.limit_check(context, self._resources, values, - project_id=project_id) + project_id=project_id, user_id=user_id) - def reserve(self, context, expire=None, project_id=None, **deltas): + def reserve(self, context, expire=None, project_id=None, user_id=None, + **deltas): """Check quotas and reserve resources. For counting quotas--those quotas for which there is a usage @@ -698,13 +944,14 @@ class QuotaEngine(object): reservations = self._driver.reserve(context, self._resources, deltas, expire=expire, - project_id=project_id) + project_id=project_id, + user_id=user_id) - LOG.debug(_("Created reservations %(reservations)s") % locals()) + LOG.debug(_("Created reservations %s"), reservations) return reservations - def commit(self, context, reservations, project_id=None): + def commit(self, context, reservations, project_id=None, user_id=None): """Commit reservations. :param context: The request context, for access checks. @@ -716,16 +963,18 @@ class QuotaEngine(object): """ try: - self._driver.commit(context, reservations, project_id=project_id) + self._driver.commit(context, reservations, project_id=project_id, + user_id=user_id) except Exception: # NOTE(Vek): Ignoring exceptions here is safe, because the # usage resynchronization and the reservation expiration # mechanisms will resolve the issue. The exception is # logged, however, because this is less than optimal. - LOG.exception(_("Failed to commit reservations " - "%(reservations)s") % locals()) + LOG.exception(_("Failed to commit reservations %s"), reservations) + return + LOG.debug(_("Committed reservations %s"), reservations) - def rollback(self, context, reservations, project_id=None): + def rollback(self, context, reservations, project_id=None, user_id=None): """Roll back reservations. :param context: The request context, for access checks. @@ -737,14 +986,47 @@ class QuotaEngine(object): """ try: - self._driver.rollback(context, reservations, project_id=project_id) + self._driver.rollback(context, reservations, project_id=project_id, + user_id=user_id) except Exception: # NOTE(Vek): Ignoring exceptions here is safe, because the # usage resynchronization and the reservation expiration # mechanisms will resolve the issue. The exception is # logged, however, because this is less than optimal. - LOG.exception(_("Failed to roll back reservations " - "%(reservations)s") % locals()) + LOG.exception(_("Failed to roll back reservations %s"), + reservations) + return + LOG.debug(_("Rolled back reservations %s"), reservations) + + def usage_reset(self, context, resources): + """ + Reset the usage records for a particular user on a list of + resources. This will force that user's usage records to be + refreshed the next time a reservation is made. + + Note: this does not affect the currently outstanding + reservations the user has; those reservations must be + committed or rolled back (or expired). + + :param context: The request context, for access checks. + :param resources: A list of the resource names for which the + usage must be reset. + """ + + self._driver.usage_reset(context, resources) + + def destroy_all_by_project_and_user(self, context, project_id, user_id): + """ + Destroy all quotas, usages, and reservations associated with a + project and user. + + :param context: The request context, for access checks. + :param project_id: The ID of the project being deleted. + :param user_id: The ID of the user being deleted. + """ + + self._driver.destroy_all_by_project_and_user(context, + project_id, user_id) def destroy_all_by_project(self, context, project_id): """ @@ -773,40 +1055,13 @@ class QuotaEngine(object): return sorted(self._resources.keys()) -def _sync_shares(context, project_id, session): - (shares, gigs) = db.share_data_get_for_project(context, - project_id, - session=session) - return {'shares': shares} - - -def _sync_snapshots(context, project_id, session): - (snapshots, gigs) = db.snapshot_data_get_for_project(context, - project_id, - session=session) - return {'snapshots': snapshots} - - -def _sync_gigabytes(context, project_id, session): - (_junk, share_gigs) = db.share_data_get_for_project(context, - project_id, - session=session) - if CONF.no_snapshot_gb_quota: - return {'gigabytes': share_gigs} - - (_junk, snap_gigs) = db.snapshot_data_get_for_project(context, - project_id, - session=session) - return {'gigabytes': share_gigs + snap_gigs} - - QUOTAS = QuotaEngine() resources = [ - ReservableResource('shares', _sync_shares, 'quota_shares'), - ReservableResource('snapshots', _sync_snapshots, 'quota_snapshots'), - ReservableResource('gigabytes', _sync_gigabytes, 'quota_gigabytes'), ] + ReservableResource('shares', '_sync_shares', 'quota_shares'), + ReservableResource('snapshots', '_sync_snapshots', 'quota_snapshots'), + ReservableResource('gigabytes', '_sync_gigabytes', 'quota_gigabytes'), ] QUOTAS.register_resources(resources) diff --git a/manila/tests/test_quota.py b/manila/tests/test_quota.py index 59e8bf4165..25da35e27f 100644 --- a/manila/tests/test_quota.py +++ b/manila/tests/test_quota.py @@ -107,6 +107,7 @@ class FakeContext(object): self.user_id = 'fake_user' self.project_id = project_id self.quota_class = quota_class + self.read_deleted = 'no' def elevated(self): elevated = self.__class__(self.project_id, self.quota_class) @@ -146,32 +147,38 @@ class FakeDriver(object): return resources def get_project_quotas(self, context, resources, project_id, - quota_class=None, defaults=True, usages=True): + quota_class=None, defaults=True, usages=True, + remains=False): self.called.append(('get_project_quotas', context, resources, - project_id, quota_class, defaults, usages)) + project_id, quota_class, defaults, usages, + remains)) return resources - def limit_check(self, context, resources, values, project_id=None): + def limit_check(self, context, resources, values, project_id=None, + user_id=None): self.called.append(('limit_check', context, resources, - values, project_id)) + values, project_id, user_id)) def reserve(self, context, resources, deltas, expire=None, - project_id=None): + project_id=None, user_id=None): self.called.append(('reserve', context, resources, deltas, - expire, project_id)) + expire, project_id, user_id)) return self.reservations - def commit(self, context, reservations, project_id=None): - self.called.append(('commit', context, reservations, project_id)) + def commit(self, context, reservations, project_id=None, user_id=None): + self.called.append(('commit', context, reservations, project_id, + user_id)) - def rollback(self, context, reservations, project_id=None): - self.called.append(('rollback', context, reservations, project_id)) + def rollback(self, context, reservations, project_id=None, user_id=None): + self.called.append(('rollback', context, reservations, project_id, + user_id)) - def delete_all_by_project(self, context, project_id): - self.called.append(('delete_all_by_project', context, project_id)) + def destroy_all_by_project_and_user(self, context, project_id, user_id): + self.called.append(('destroy_all_by_project_and_user', context, + project_id, user_id)) def destroy_all_by_project(self, context, project_id): - self.called.append(('delete_all_by_project', context, project_id)) + self.called.append(('destroy_all_by_project', context, project_id)) def expire(self, context): self.called.append(('expire', context)) @@ -425,13 +432,15 @@ class QuotaEngineTestCase(test.TestCase): 'test_project', None, True, - True), + True, + False), ('get_project_quotas', context, quota_obj._resources, 'test_project', 'test_class', False, + False, False), ]) self.assertEqual(result1, quota_obj._resources) self.assertEqual(result2, quota_obj._resources) @@ -483,7 +492,7 @@ class QuotaEngineTestCase(test.TestCase): test_resource2=3, test_resource3=2, test_resource4=1,), - None), ]) + None, None), ]) def test_reserve(self): context = FakeContext(None, None) @@ -512,6 +521,7 @@ class QuotaEngineTestCase(test.TestCase): test_resource3=2, test_resource4=1, ), None, + None, None), ('reserve', context, @@ -522,6 +532,7 @@ class QuotaEngineTestCase(test.TestCase): test_resource3=3, test_resource4=4, ), 3600, + None, None), ('reserve', context, @@ -532,7 +543,7 @@ class QuotaEngineTestCase(test.TestCase): test_resource3=3, test_resource4=4, ), None, - 'fake_project'), ]) + 'fake_project', None), ]) self.assertEqual(result1, ['resv-01', 'resv-02', 'resv-03', @@ -558,7 +569,7 @@ class QuotaEngineTestCase(test.TestCase): ['resv-01', 'resv-02', 'resv-03'], - None), ]) + None, None), ]) def test_rollback(self): context = FakeContext(None, None) @@ -572,16 +583,28 @@ class QuotaEngineTestCase(test.TestCase): ['resv-01', 'resv-02', 'resv-03'], - None), ]) + None, None), ]) - def test_delete_all_by_project(self): + def test_destroy_all_by_project_and_user(self): + context = FakeContext(None, None) + driver = FakeDriver() + quota_obj = self._make_quota_obj(driver) + quota_obj.destroy_all_by_project_and_user(context, + 'test_project', 'fake_user') + + self.assertEqual(driver.called, [ + ('destroy_all_by_project_and_user', context, 'test_project', + 'fake_user'), + ]) + + def test_destroy_all_by_project(self): context = FakeContext(None, None) driver = FakeDriver() quota_obj = self._make_quota_obj(driver) quota_obj.destroy_all_by_project(context, 'test_project') self.assertEqual(driver.called, - [('delete_all_by_project', + [('destroy_all_by_project', context, 'test_project'), ]) @@ -661,6 +684,55 @@ class DbQuotaDriverTestCase(test.TestCase): self.assertEqual(result, dict(shares=10, gigabytes=500)) + def _stub_get_by_project_and_user(self): + def fake_qgabpu(context, project_id, user_id): + self.calls.append('quota_get_all_by_project_and_user') + self.assertEqual(project_id, 'test_project') + self.assertEqual(user_id, 'fake_user') + return dict(shares=10, gigabytes=50, reserved=0) + + def fake_qgabp(context, project_id): + self.calls.append('quota_get_all_by_project') + self.assertEqual(project_id, 'test_project') + return dict(shares=10, gigabytes=50, reserved=0) + + def fake_qugabpu(context, project_id, user_id): + self.calls.append('quota_usage_get_all_by_project_and_user') + self.assertEqual(project_id, 'test_project') + self.assertEqual(user_id, 'fake_user') + return dict(shares=dict(in_use=2, reserved=0), + gigabytes=dict(in_use=10, reserved=0), ) + + self.stubs.Set(db, 'quota_get_all_by_project_and_user', fake_qgabpu) + self.stubs.Set(db, 'quota_get_all_by_project', fake_qgabp) + self.stubs.Set(db, 'quota_usage_get_all_by_project_and_user', + fake_qugabpu) + + self._stub_quota_class_get_all_by_name() + + def test_get_user_quotas(self): + self._stub_get_by_project_and_user() + result = self.driver.get_user_quotas( + FakeContext('test_project', 'test_class'), + quota.QUOTAS._resources, 'test_project', 'fake_user') + + self.assertEqual(self.calls, [ + 'quota_get_all_by_project_and_user', + 'quota_get_all_by_project', + 'quota_usage_get_all_by_project_and_user', + 'quota_class_get_all_by_name', + ]) + self.assertEqual(result, dict(shares=dict(limit=10, + in_use=2, + reserved=0, ), + gigabytes=dict(limit=50, + in_use=10, + reserved=0, ), + snapshots=dict(limit=10, + in_use=0, + reserved=0, ), + )) + def _stub_get_by_project(self): def fake_qgabp(context, project_id): self.calls.append('quota_get_all_by_project') @@ -698,10 +770,32 @@ class DbQuotaDriverTestCase(test.TestCase): reserved=0, ), )) + def test_get_user_quotas_alt_context_no_class(self): + self._stub_get_by_project_and_user() + result = self.driver.get_user_quotas( + FakeContext('other_project', None), + quota.QUOTAS._resources, 'test_project', 'fake_user') + + self.assertEqual(self.calls, [ + 'quota_get_all_by_project_and_user', + 'quota_get_all_by_project', + 'quota_usage_get_all_by_project_and_user', + ]) + self.assertEqual(result, dict(shares=dict(limit=10, + in_use=2, + reserved=0, ), + gigabytes=dict(limit=50, + in_use=10, + reserved=0, ), + snapshots=dict(limit=10, + in_use=0, + reserved=0, ), + )) + def test_get_project_quotas_alt_context_no_class(self): self._stub_get_by_project() result = self.driver.get_project_quotas( - FakeContext('other_project', 'other_class'), + FakeContext('other_project', None), quota.QUOTAS._resources, 'test_project') self.assertEqual(self.calls, ['quota_get_all_by_project', @@ -717,6 +811,30 @@ class DbQuotaDriverTestCase(test.TestCase): reserved=0, ), )) + def test_get_user_quotas_alt_context_with_class(self): + self._stub_get_by_project_and_user() + result = self.driver.get_user_quotas( + FakeContext('other_project', 'other_class'), + quota.QUOTAS._resources, 'test_project', 'fake_user', + quota_class='test_class') + + self.assertEqual(self.calls, [ + 'quota_get_all_by_project_and_user', + 'quota_get_all_by_project', + 'quota_usage_get_all_by_project_and_user', + 'quota_class_get_all_by_name', + ]) + self.assertEqual(result, dict(shares=dict(limit=10, + in_use=2, + reserved=0, ), + gigabytes=dict(limit=50, + in_use=10, + reserved=0, ), + snapshots=dict(limit=10, + in_use=0, + reserved=0, ), + )) + def test_get_project_quotas_alt_context_with_class(self): self._stub_get_by_project() result = self.driver.get_project_quotas( @@ -737,6 +855,27 @@ class DbQuotaDriverTestCase(test.TestCase): reserved=0, ), )) + def test_get_user_quotas_no_defaults(self): + self._stub_get_by_project_and_user() + result = self.driver.get_user_quotas( + FakeContext('test_project', 'test_class'), + quota.QUOTAS._resources, 'test_project', 'fake_user', + defaults=False) + + self.assertEqual(self.calls, [ + 'quota_get_all_by_project_and_user', + 'quota_get_all_by_project', + 'quota_usage_get_all_by_project_and_user', + 'quota_class_get_all_by_name', + ]) + self.assertEqual(result, + dict(gigabytes=dict(limit=50, + in_use=10, + reserved=0, ), + shares=dict(limit=10, + in_use=2, + reserved=0, ), )) + def test_get_project_quotas_no_defaults(self): self._stub_get_by_project() result = self.driver.get_project_quotas( @@ -754,6 +893,21 @@ class DbQuotaDriverTestCase(test.TestCase): in_use=2, reserved=0, ), )) + def test_get_user_quotas_no_usages(self): + self._stub_get_by_project_and_user() + result = self.driver.get_user_quotas( + FakeContext('test_project', 'test_class'), + quota.QUOTAS._resources, 'test_project', 'fake_user', usages=False) + + self.assertEqual(self.calls, [ + 'quota_get_all_by_project_and_user', + 'quota_get_all_by_project', + 'quota_class_get_all_by_name', + ]) + self.assertEqual(result, dict(shares=dict(limit=10, ), + gigabytes=dict(limit=50, ), + snapshots=dict(limit=10))) + def test_get_project_quotas_no_usages(self): self._stub_get_by_project() result = self.driver.get_project_quotas( @@ -766,6 +920,77 @@ class DbQuotaDriverTestCase(test.TestCase): gigabytes=dict(limit=50, ), snapshots=dict(limit=10))) + def _stub_get_settable_quotas(self): + def fake_get_project_quotas(context, resources, project_id, + quota_class=None, defaults=True, + usages=True, remains=False): + self.calls.append('get_project_quotas') + result = {} + for k, v in resources.items(): + remains = v.default + in_use = 0 + result[k] = {'limit': v.default, 'in_use': in_use, + 'reserved': 0, 'remains': remains} + return result + + def fake_get_user_quotas(context, resources, project_id, user_id, + quota_class=None, defaults=True, + usages=True): + self.calls.append('get_user_quotas') + result = {} + for k, v in resources.items(): + in_use = 0 + result[k] = {'limit': v.default, + 'in_use': in_use, 'reserved': 0} + return result + + def fake_qgabpau(context, project_id, user_id): + self.calls.append('quota_get_all_by_project_and_user') + return {'shares': 2} + + self.stubs.Set(self.driver, 'get_project_quotas', + fake_get_project_quotas) + self.stubs.Set(self.driver, 'get_user_quotas', + fake_get_user_quotas) + self.stubs.Set(db, 'quota_get_all_by_project_and_user', + fake_qgabpau) + + def test_get_settable_quotas_with_user(self): + self._stub_get_settable_quotas() + result = self.driver.get_settable_quotas( + FakeContext('test_project', 'test_class'), + quota.QUOTAS._resources, 'test_project', user_id='test_user') + + self.assertEqual(self.calls, [ + 'get_project_quotas', + 'get_user_quotas', + 'quota_get_all_by_project_and_user', + ]) + self.assertEqual(result, dict(shares=dict(maximum=12, + minimum=0, ), + gigabytes=dict(maximum=1000, + minimum=0, ), + snapshots=dict(maximum=10, + minimum=0, ), + )) + + def test_get_settable_quotas_without_user(self): + self._stub_get_settable_quotas() + result = self.driver.get_settable_quotas( + FakeContext('test_project', 'test_class'), + quota.QUOTAS._resources, 'test_project') + + self.assertEqual(self.calls, [ + 'get_project_quotas', + ]) + self.assertEqual(result, dict(shares=dict(maximum=-1, + minimum=0, ), + gigabytes=dict(maximum=-1, + minimum=0, ), + snapshots=dict(maximum=-1, + minimum=0, ), + )) + def _stub_get_project_quotas(self): def fake_get_project_quotas(context, resources, project_id, quota_class=None, defaults=True, @@ -821,8 +1046,9 @@ class DbQuotaDriverTestCase(test.TestCase): self.assertEqual(result, dict(shares=10, gigabytes=1000, )) def _stub_quota_reserve(self): - def fake_quota_reserve(context, resources, quotas, deltas, expire, - until_refresh, max_age, project_id=None): + def fake_quota_reserve(context, resources, quotas, user_quotas, + deltas, expire, until_refresh, max_age, + project_id=None, user_id=None): self.calls.append(('quota_reserve', expire, until_refresh, max_age)) return ['resv-1', 'resv-2', 'resv-3'] @@ -933,6 +1159,9 @@ class FakeSession(object): def begin(self): return self + def add(self, instance): + pass + def __enter__(self): return self @@ -956,7 +1185,7 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): self.sync_called = set() def make_sync(res_name): - def sync(context, project_id, session): + def sync(context, project_id, user_id, session): self.sync_called.add(res_name) if res_name in self.usages: if self.usages[res_name].in_use < 0: @@ -968,7 +1197,9 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): self.resources = {} for res_name in ('shares', 'gigabytes'): - res = quota.ReservableResource(res_name, make_sync(res_name)) + method_name = '_sync_%s' % res_name + sqa_api.QUOTA_SYNC_FUNCTIONS[method_name] = make_sync(res_name) + res = quota.ReservableResource(res_name, '_sync_%s' % res_name) self.resources[res_name] = res self.expire = timeutils.utcnow() + datetime.timedelta(seconds=3600) @@ -980,14 +1211,17 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): def fake_get_session(): return FakeSession() - def fake_get_quota_usages(context, session, project_id): + def fake_get_project_quota_usages(context, session, project_id): return self.usages.copy() - def fake_quota_usage_create(context, project_id, resource, in_use, - reserved, until_refresh, session=None, - save=True): + def fake_get_user_quota_usages(context, session, project_id, user_id): + return self.usages.copy() + + def fake_quota_usage_create(context, project_id, user_id, resource, + in_use, reserved, until_refresh, + session=None, save=True): quota_usage_ref = self._make_quota_usage( - project_id, resource, in_use, reserved, until_refresh, + project_id, user_id, resource, in_use, reserved, until_refresh, timeutils.utcnow(), timeutils.utcnow()) self.usages_created[resource] = quota_usage_ref @@ -995,9 +1229,10 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): return quota_usage_ref def fake_reservation_create(context, uuid, usage_id, project_id, - resource, delta, expire, session=None): + user_id, resource, delta, expire, + session=None): reservation_ref = self._make_reservation( - uuid, usage_id, project_id, resource, delta, expire, + uuid, usage_id, project_id, user_id, resource, delta, expire, timeutils.utcnow(), timeutils.utcnow()) self.reservations_created[resource] = reservation_ref @@ -1005,14 +1240,17 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): return reservation_ref self.stubs.Set(sqa_api, 'get_session', fake_get_session) - self.stubs.Set(sqa_api, '_get_quota_usages', fake_get_quota_usages) - self.stubs.Set(sqa_api, 'quota_usage_create', fake_quota_usage_create) - self.stubs.Set(sqa_api, 'reservation_create', fake_reservation_create) + self.stubs.Set(sqa_api, '_get_project_quota_usages', + fake_get_project_quota_usages) + self.stubs.Set(sqa_api, '_get_user_quota_usages', + fake_get_user_quota_usages) + self.stubs.Set(sqa_api, '_quota_usage_create', fake_quota_usage_create) + self.stubs.Set(sqa_api, '_reservation_create', fake_reservation_create) timeutils.set_time_override() - def _make_quota_usage(self, project_id, resource, in_use, reserved, - until_refresh, created_at, updated_at): + def _make_quota_usage(self, project_id, user_id, resource, in_use, + reserved, until_refresh, created_at, updated_at): quota_usage_ref = FakeUsage() quota_usage_ref.id = len(self.usages) + len(self.usages_created) quota_usage_ref.project_id = project_id @@ -1027,14 +1265,15 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): return quota_usage_ref - def init_usage(self, project_id, resource, in_use, reserved, + def init_usage(self, project_id, user_id, resource, in_use, reserved, until_refresh=None, created_at=None, updated_at=None): if created_at is None: created_at = timeutils.utcnow() if updated_at is None: updated_at = timeutils.utcnow() - quota_usage_ref = self._make_quota_usage(project_id, resource, in_use, + quota_usage_ref = self._make_quota_usage(project_id, user_id, + resource, in_use, reserved, until_refresh, created_at, updated_at) @@ -1049,7 +1288,7 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): "%s != %s on usage for resource %s" % (actual, value, resource)) - def _make_reservation(self, uuid, usage_id, project_id, resource, + def _make_reservation(self, uuid, usage_id, project_id, user_id, resource, delta, expire, created_at, updated_at): reservation_ref = sqa_models.Reservation() reservation_ref.id = len(self.reservations_created) @@ -1090,7 +1329,7 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): deltas = dict(shares=2, gigabytes=2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 0, 0) + quotas, deltas, self.expire, 0, 0) self.assertEqual(self.sync_called, set(['shares', 'gigabytes'])) self.compare_usage(self.usages_created, @@ -1115,15 +1354,17 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): delta=2 * 1024), ]) def test_quota_reserve_negative_in_use(self): - self.init_usage('test_project', 'shares', -1, 0, until_refresh=1) - self.init_usage('test_project', 'gigabytes', -1, 0, until_refresh=1) + self.init_usage('test_project', 'test_user', 'shares', -1, 0, + until_refresh=1) + self.init_usage('test_project', 'test_user', 'gigabytes', -1, 0, + until_refresh=1) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=2, gigabytes=2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 5, 0) + quotas, deltas, self.expire, 5, 0) self.assertEqual(self.sync_called, set(['shares', 'gigabytes'])) self.compare_usage(self.usages, [dict(resource='shares', @@ -1147,13 +1388,15 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): delta=2 * 1024), ]) def test_quota_reserve_until_refresh(self): - self.init_usage('test_project', 'shares', 3, 0, until_refresh=1) - self.init_usage('test_project', 'gigabytes', 3, 0, until_refresh=1) + self.init_usage('test_project', 'test_user', 'shares', 3, 0, + until_refresh=1) + self.init_usage('test_project', 'test_user', 'gigabytes', 3, 0, + until_refresh=1) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=2, gigabytes=2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 5, 0) + quotas, deltas, self.expire, 5, 0) self.assertEqual(self.sync_called, set(['shares', 'gigabytes'])) self.compare_usage(self.usages, [dict(resource='shares', @@ -1180,15 +1423,16 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): max_age = 3600 record_created = (timeutils.utcnow() - datetime.timedelta(seconds=max_age)) - self.init_usage('test_project', 'shares', 3, 0, + self.init_usage('test_project', 'test_user', 'shares', 3, 0, created_at=record_created, updated_at=record_created) - self.init_usage('test_project', 'gigabytes', 3, 0, + self.init_usage('test_project', 'test_user', 'gigabytes', 3, 0, created_at=record_created, updated_at=record_created) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=2, gigabytes=2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 0, max_age) + quotas, deltas, self.expire, 0, + max_age) self.assertEqual(self.sync_called, set(['shares', 'gigabytes'])) self.compare_usage(self.usages, [dict(resource='shares', @@ -1212,13 +1456,13 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): delta=2 * 1024), ]) def test_quota_reserve_no_refresh(self): - self.init_usage('test_project', 'shares', 3, 0) - self.init_usage('test_project', 'gigabytes', 3, 0) + self.init_usage('test_project', 'test_user', 'shares', 3, 0) + self.init_usage('test_project', 'test_user', 'gigabytes', 3, 0) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=2, gigabytes=2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 0, 0) + quotas, deltas, self.expire, 0, 0) self.assertEqual(self.sync_called, set([])) self.compare_usage(self.usages, [dict(resource='shares', @@ -1242,13 +1486,13 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): delta=2 * 1024), ]) def test_quota_reserve_unders(self): - self.init_usage('test_project', 'shares', 1, 0) - self.init_usage('test_project', 'gigabytes', 1 * 1024, 0) + self.init_usage('test_project', 'test_user', 'shares', 1, 0) + self.init_usage('test_project', 'test_user', 'gigabytes', 1 * 1024, 0) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=-2, gigabytes=-2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 0, 0) + quotas, deltas, self.expire, 0, 0) self.assertEqual(self.sync_called, set([])) self.compare_usage(self.usages, [dict(resource='shares', @@ -1272,14 +1516,15 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): delta=-2 * 1024), ]) def test_quota_reserve_overs(self): - self.init_usage('test_project', 'shares', 4, 0) - self.init_usage('test_project', 'gigabytes', 10 * 1024, 0) + self.init_usage('test_project', 'test_user', 'shares', 4, 0) + self.init_usage('test_project', 'test_user', 'gigabytes', 10 * 1024, + 0) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=2, gigabytes=2 * 1024, ) self.assertRaises(exception.OverQuota, sqa_api.quota_reserve, - context, self.resources, quotas, + context, self.resources, quotas, quotas, deltas, self.expire, 0, 0) self.assertEqual(self.sync_called, set([])) @@ -1297,13 +1542,14 @@ class QuotaReserveSqlAlchemyTestCase(test.TestCase): self.assertEqual(self.reservations_created, {}) def test_quota_reserve_reduction(self): - self.init_usage('test_project', 'shares', 10, 0) - self.init_usage('test_project', 'gigabytes', 20 * 1024, 0) + self.init_usage('test_project', 'test_user', 'shares', 10, 0) + self.init_usage('test_project', 'test_user', 'gigabytes', 20 * 1024, + 0) context = FakeContext('test_project', 'test_class') quotas = dict(shares=5, gigabytes=10 * 1024, ) deltas = dict(shares=-2, gigabytes=-2 * 1024, ) result = sqa_api.quota_reserve(context, self.resources, quotas, - deltas, self.expire, 0, 0) + quotas, deltas, self.expire, 0, 0) self.assertEqual(self.sync_called, set([])) self.compare_usage(self.usages, [dict(resource='shares',