openstack/glance
Code Issues Proposed changes

Add image_count_uploading quota enforcement

Browse Source 
This makes us enforce a quota on the total number of upload-related
image operations owned by a user.

Partially-implements: blueprint glance-unified-quotas

Change-Id: I2a28750aaf968e6a6324eb194d4280a640bfa5aa
This commit is contained in:
Dan Smith 2021-06-01 10:02:24 -07:00
parent 084a77e644
commit 8d6ee6f822
6 changed files with 123 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
glance/api/v2/image_data.py
View File

@ -169,6 +169,8 @@ class ImageDataController(object):
encodeutils.exception_to_unicode(e))
image_repo.save(image, from_state='queued')
ks_quota.enforce_image_count_uploading(req.context,
req.context.owner)
image.set_data(data, size, backend=backend)
try:
@ -275,6 +277,12 @@ class ImageDataController(object):
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.LimitExceeded as e:
LOG.error(str(e))
self._restore(image_repo, image)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e),
request=req)
except glance_store.StorageWriteDenied as e:
msg = _("Insufficient permissions on image "
"storage media: %s") % encodeutils.exception_to_unicode(e)
@ -357,6 +365,8 @@ class ImageDataController(object):
image = image_repo.get(image_id)
image.status = 'uploading'
image_repo.save(image, from_state='queued')
ks_quota.enforce_image_count_uploading(req.context,
req.context.owner)
try:
uri, size, id, store_info = staging_store.add(
image_id, utils.LimitingReader(
@ -401,6 +411,12 @@ class ImageDataController(object):
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req)
except exception.LimitExceeded as e:
LOG.debug(str(e))
self._unstage(image_repo, image, staging_store)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e),
request=req)
except glance_store.StorageWriteDenied as e:
msg = _("Insufficient permissions on image "
"storage media: %s") % encodeutils.exception_to_unicode(e)

3
glance/api/v2/images.py
View File

@ -469,6 +469,9 @@ class ImagesController(object):
raise webob.exc.HTTPConflict(explanation=e.msg)
except exception.InvalidImageStatusTransition as e:
raise webob.exc.HTTPConflict(explanation=e.msg)
except exception.LimitExceeded as e:
raise webob.exc.HTTPRequestEntityTooLarge(explanation=str(e),
request=req)
except ValueError as e:
LOG.debug("Cannot import data for image %(id)s: %(e)s",
{'id': image_id,

3
glance/async_/flows/api_image_import.py
View File

@ -882,5 +882,8 @@ def get_flow(**kwargs):
stores, action_wrapper,
ks_quota.enforce_image_staging_total,
delta=image_size)
assert_quota(kwargs['context'], task_repo, task_id,
stores, action_wrapper,
ks_quota.enforce_image_count_uploading)
return flow

2
glance/async_/taskflow_executor.py
View File

@ -137,6 +137,8 @@ class TaskExecutor(glance.async_.TaskExecutor):
raise exception.ImportTaskError(message=exc.reason)
except (exception.BadStoreUri, exception.Invalid) as exc:
raise exception.ImportTaskError(message=exc.msg)
except exception.LimitExceeded as exc:
raise exception.ImportTaskError(message=exc.msg)
except RuntimeError:
raise NotImplementedError()
except Exception as e:

17
glance/quota/keystone.py
View File

@ -30,6 +30,7 @@ limit.opts.register_opts(CONF)
QUOTA_IMAGE_SIZE_TOTAL = 'image_size_total'
QUOTA_IMAGE_STAGING_TOTAL = 'image_stage_total'
QUOTA_IMAGE_COUNT_TOTAL = 'image_count_total'
QUOTA_IMAGE_COUNT_UPLOADING = 'image_count_uploading'
def _enforce_some(context, project_id, quota_value_fns, deltas):
@ -125,3 +126,19 @@ def enforce_image_count_total(context, project_id):
context, project_id, QUOTA_IMAGE_COUNT_TOTAL,
lambda: db.user_get_image_count(context, project_id),
delta=1)
def enforce_image_count_uploading(context, project_id):
"""Enforce the image_count_uploading quota.
This enforces the total count of images in any state of upload by
the supplied project_id.
:param delta: This defaults to one, but should be zero when checking
an operation on an image that already counts against this
quota (i.e. a stage operation of an existing queue image).
"""
_enforce_one(
context, project_id, QUOTA_IMAGE_COUNT_UPLOADING,
lambda: db.user_get_uploading_count(context, project_id),
delta=0)

90
glance/tests/functional/v2/test_images.py
View File

@ -7074,7 +7074,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
def test_upload(self):
# Set a quota of 5MiB
self.set_limit({'image_size_total': 5,
'image_count_total': 10})
'image_count_total': 10,
'image_count_uploading': 10})
self.start_server()
# First upload of 3MiB is good
@ -7098,7 +7099,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
def test_import(self):
# Set a quota of 5MiB
self.set_limit({'image_size_total': 5,
'image_count_total': 10})
'image_count_total': 10,
'image_count_uploading': 10})
self.start_server()
# First upload of 3MiB is good
@ -7121,7 +7123,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
def test_import_would_go_over(self):
# Set a quota limit of 5MiB
self.set_limit({'image_size_total': 5,
'image_count_total': 10})
'image_count_total': 10,
'image_count_uploading': 10})
self.start_server()
# First upload of 3MiB is good
@ -7161,7 +7164,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
# Set a size quota of 5MiB, with more staging quota than we need.
self.set_limit({'image_size_total': 5,
'image_count_total': 10,
'image_stage_total': 15})
'image_stage_total': 15,
'image_count_uploading': 10})
self.start_server()
# First import of 3MiB is good
@ -7185,7 +7189,9 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
# before copy. This request should succeed, but the copy task
# should fail the staging quota check.
self.set_limit({'image_size_total': 15,
'image_stage_total': 5})
'image_count_total': 10,
'image_stage_total': 5,
'image_count_uploading': 10})
req = self._import_copy(image_id, ['store3'])
self.assertEqual(202, req.status_code)
self._wait_for_import(image_id)
@ -7193,7 +7199,9 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
# If we increase our stage quota, we should now be able to copy.
self.set_limit({'image_size_total': 15,
'image_stage_total': 10})
'image_count_total': 10,
'image_stage_total': 10,
'image_count_uploading': 10})
req = self._import_copy(image_id, ['store3'])
self.assertEqual(202, req.status_code)
self._wait_for_import(image_id)
@ -7203,7 +7211,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
# Set a quota of 5MiB
self.set_limit({'image_size_total': 15,
'image_stage_total': 5,
'image_count_total': 10})
'image_count_total': 10,
'image_count_uploading': 10})
self.start_server()
# Stage 6MiB, which is allowed to complete, but leaves us over
@ -7237,7 +7246,8 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
def test_create(self):
# Set a quota of 2 images
self.set_limit({'image_size_total': 15,
'image_count_total': 2})
'image_count_total': 2,
'image_count_uploading': 10})
self.start_server()
# Create one image
@ -7255,3 +7265,67 @@ class TestKeystoneQuotas(functional.SynchronousAPIBase):
# Now we can create that third image
self._create()
def test_uploading_methods(self):
self.set_limit({'image_size_total': 100,
'image_stage_total': 100,
'image_count_total': 100,
'image_count_uploading': 1})
self.start_server()
# Create and stage one image. We are now at quota for count_uploading.
image_id = self._create_and_stage()
# Make sure we can not stage any more images.
self._create_and_stage(expected_code=413)
# Make sure we can not upload any more images.
self._create_and_upload(expected_code=413)
# Finish importing one of the images, which should put us under quota
# for count_uploading.
resp = self._import_direct(image_id, ['store1'])
self.assertEqual(202, resp.status_code)
self.assertEqual('active', self._wait_for_import(image_id)['status'])
# Make sure we can upload now.
self._create_and_upload()
# Stage another, which should put us at quota for count_uploading.
image_id2 = self._create_and_stage()
# Start a copy. The request should succeed (because async) but
# the task should ultimately fail because we are over quota.
# NOTE(danms): It would be nice to try to do another copy or
# upload while this is running, but since the task is fully
# async and the copy happens quickly, we can't really time it
# to avoid an unstable test (without some mocking).
resp = self._import_copy(image_id, ['store2'])
self.assertEqual(202, resp.status_code)
self._wait_for_import(image_id)
task = self._get_latest_task(image_id)
self.assertEqual('failure', task['status'])
self.assertIn('Resource image_count_uploading is over limit',
task['message'])
# Finish the staged import.
self._import_direct(image_id2, ['store1'])
self.assertEqual(202, resp.status_code)
self._wait_for_import(image_id2)
# Make sure we can upload again after the import finishes.
self._create_and_upload()
# Re-try the copy that should now succeed and wait for it to
# finish.
resp = self._import_copy(image_id, ['store2'])
self.assertEqual(202, resp.status_code)
self._wait_for_import(image_id)
task = self._get_latest_task(image_id)
self.assertEqual('success', task['status'])
# Make sure we can still upload.
self._create_and_upload()
# Make sure we can still import.
self._create_and_import(stores=['store1'])